diff --git a/tierb_study_20260826_144644/.completed/echo__tierbprobe__gbs8 b/tierb_study_20260826_144644/.completed/echo__tierbprobe__gbs8 index 8b9892f0e38737b7440a83d5bc737fadf61a7e3f..df6cf376fb60eee0b69bbd9b5c93981c6c64c54c 100644 --- a/tierb_study_20260826_144644/.completed/echo__tierbprobe__gbs8 +++ b/tierb_study_20260826_144644/.completed/echo__tierbprobe__gbs8 @@ -1 +1 @@ -2026-08-26T14:57:54+00:00 +2026-08-26T15:43:24+00:00 diff --git a/tierb_study_20260826_144644/STATUS.txt b/tierb_study_20260826_144644/STATUS.txt index 7da3fa4df198a0fefe876e83313aa6e4b4aa3415..2fd107bcb9bb54d67f3f8a74e879dc30abd61fd5 100644 --- a/tierb_study_20260826_144644/STATUS.txt +++ b/tierb_study_20260826_144644/STATUS.txt @@ -1,9 +1,10 @@ study_root: /workspace/tierb_study_20260826_144644 gbs: 8 -updated: 2026-08-26T15:32:33+00:00 -current: echo__tierbprobe__gbs8 +updated: 2026-08-26T15:43:26+00:00 +current: ultraep__tierbprobe__gbs8 completed steps: + echo__tierbprobe__gbs8 failed steps: FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe diff --git a/tierb_study_20260826_144644/hf_watcher.log b/tierb_study_20260826_144644/hf_watcher.log index fc738ca5f1fbade94a88edf9fa19dab7b6891fe1..b2bbf1d0c3af73f473e6e608a99dea64aa890706 100644 --- a/tierb_study_20260826_144644/hf_watcher.log +++ b/tierb_study_20260826_144644/hf_watcher.log @@ -20,3 +20,6 @@ Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop) uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644 https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main [2026-08-26T15:42:37+00:00] backup pass + uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644 +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +[2026-08-26T15:52:40+00:00] backup pass diff --git a/tierb_study_20260826_144644/steps.tsv b/tierb_study_20260826_144644/steps.tsv index fbb2586c08f5fdcf032dfda7d3af9289962f0c57..231261f3513c441869ee05d04b28abac61e39000 100644 --- a/tierb_study_20260826_144644/steps.tsv +++ b/tierb_study_20260826_144644/steps.tsv @@ -2,3 +2,4 @@ status campaign phase gbs started ended result_dir OK echo tierbprobe 8 2026-08-26T14:46:44+00:00 2026-08-26T14:57:54+00:00 /workspace/tierb_study_20260826_144644/echo/tierbprobe FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe FAIL moonep tierbprobe 8 2026-08-26T15:05:24+00:00 2026-08-26T15:12:52+00:00 /workspace/tierb_study_20260826_144644/moonep/tierbprobe +OK echo tierbprobe 8 2026-08-26T15:32:33+00:00 2026-08-26T15:43:24+00:00 /workspace/tierb_study_20260826_144644/echo/tierbprobe diff --git a/tierb_study_20260826_144644/study.log b/tierb_study_20260826_144644/study.log index db918bbbd490cfd21fd5cbff72ce11ef60b9a05b..cd9617bd3e3b464ae0c5e314a68e769d2529308c 100644 --- a/tierb_study_20260826_144644/study.log +++ b/tierb_study_20260826_144644/study.log @@ -503,3 +503,135 @@ driver_repo: /workspace/blackwell_run_scripts @ d11391f [guard] tearing down process group (outcome=DONE) -------------------------------------------------------------------------- [15:42:07] CELL QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal (slots=8) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- + +============================================================== + SWEEP COMPLETE ok=8 failed=0 +============================================================== +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 441.20 435.60 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN3_235B echo matched 16 8192 8 2.00 none+forcebal 1 OK 422.89 416.80 141939 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 +QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 1052.93 996.60 133569 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN3_30B echo matched 16 4096 8 2.00 none+forcebal 1 OK 963.67 920.30 133533 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 +DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 806.56 800.50 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 +DSV2_LITE_L24 echo matched 8 8192 8 2.00 none+forcebal 1 OK 799.97 793.80 140541 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 +QWEN2_57B echo matched 8 4096 8 2.00 none 1 OK 286.46 282.70 109133 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN2_57B echo matched 8 4096 8 2.00 none+forcebal 1 OK 276.56 271.10 109131 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 + + Results: /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv + Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8 +============================================================== +8 probe runs -> /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +run arm cap rawExpI preRankI postRankI drop% reqCap +-------------------------------------------------------------------------------------------------------------------- +DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 natural 2.00 7.4425* 2.1173 1.0000 1.0000 +DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 forced 2.00 1.0287 1.0061 1.0000 1.0000 +QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep natural 2.00 6.8085* 2.1623 1.0000 1.0000 +QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__fo forced 2.00 1.0352 1.0074 1.0000 1.0000 +QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/re natural 2.00 15.4584* 2.5319 1.0000 1.0000 +QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__f forced 2.00 1.0390 1.0053 1.0000 1.0000 +QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep natural 2.00 14.5598* 2.3229 1.0000 1.0000 +QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__fo forced 2.00 1.0565 1.0077 1.0000 1.0000 + + * At least one globally aggregated snapshot reached the E/K expert-Imax ceiling. + Inspect raw_expert_imax_mean and hottest_expert_token_fraction_mean before generalising. + +ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional. +The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally. + + TIER B PROBE -- what these logs contain, per backend + ---------------------------------------------------- + Capacity matrix: + ECHO: start C=2.00; on OOM descend 1.75, 1.50, 1.25. Stop at + the highest fitting zero-drop capacity. If a fitting capacity drops, + smaller values cannot help and the model is blocked. + UltraEP: measure required_capacity at C=1.25, round it UP to the next + 0.01, then validate that capacity once. + MoonEP: natural + forced, no capacity factor. + + The forced arm runs once at the selected/high-headroom capacity. It validates + instrumentation and is not part of capacity selection. + + Each backend instruments a different thing: + + echo local raw-router histograms and local post-reroute placement. + reduce_probe.py aligns and sums all eight source ranks before + calculating raw_expert_*, pre_home_rank_*, + post_physical_rank_* and valid_max_rank_reduction_*. + + ultraep global pre/post rank vectors plus explicit drop accounting: + drop_rate_pct and required_capacity -- the capacity factor the run + WOULD have needed. The vectors are duplicated identically on all + logging ranks and are deduplicated, not summed. + + moonep local raw-router histograms, actual payload rows per physical + destination, and the bytes-copy side. The reducer globally sums + each (iteration, layer) snapshot first. ECHO reports no comparable + byte counters, so MoonEP bytes are a self-comparison across models, + never a cross-backend ranking. + + Do NOT quote ECHO's logged pre_echo imbalance_ratio or + max_load_reduction_pct. The branch mixes source-rank and destination-rank + axes; the reducer marks both invalid and reconstructs the comparison globally. + + Two boundary cases: + raw_expert_imax_max == E/K + -> at least one global snapshot had one expert in every + token's top-K set. Inspect the MEAN before generalising. + required_capacity or min-safe-capacity at the configured capacity + -> clipped by the buffer; it means "at or over", not + "fits comfortably". + + tools/reduce_probe.py has already written probe_summary.tsv and .json here. + + Campaign results: /workspace/tierb_study_20260826_144644/echo/tierbprobe + Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tierb_study_20260826_144644/echo/tierbprobe --compare +Already logged in to Hugging Face as: AbhishekGhosh1998 + uploaded: /workspace/tierb_study_20260826_144644/echo/tierbprobe -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644/echo/tierbprobe +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main + +=== STEP ultraep__tierbprobe__gbs8 +============================================================== + CAMPAIGN: ultraep backend=ultraep phase=tierbprobe + Megatron: /workspace/Megatron-UltraEP @ ultraep-token-drop-analysis + GBS=8 seq=4096 repeats=3 nsys=1 + results -> /workspace/tierb_study_20260826_144644/ultraep/tierbprobe +============================================================== +-------------------------------------------------------------------------- +[15:43:42] CELL QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25 (slots=16) + [guard] iter 9/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:44:54] CELL QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal (slots=16) + [guard] iter 12/12 (60s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:46:01] CELL QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25 (slots=16) + [guard] iter 4/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:47:16] CELL QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal (slots=16) + [guard] iter 3/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:48:32] CELL DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25 (slots=8) + [guard] iter 4/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:49:47] CELL DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal (slots=8) + [guard] iter 5/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:51:01] CELL QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25 (slots=8) + [guard] iter 12/12 (60s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:52:03] CELL QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal (slots=8) +[15:52:06] waiting for stale ranks to exit ... diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt b/tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt index 8666098ab488ef5a2919e471a36bc86c88bac72e..e9345d1e499a0e67a8f431d289070c6608090e6a 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/manifest.txt @@ -1,9 +1,9 @@ -timestamp: 2026-08-26T14:58:12+00:00 +timestamp: 2026-08-26T15:43:41+00:00 host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 gpu: NVIDIA B200 x8 driver: 580.95.05 study_scripts_branch: granularity-ultraep -study_scripts_commit: 81efb04 +study_scripts_commit: 05e7365 campaign: ultraep backend: ultraep phase: tierbprobe diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt b/tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt index 4b7329475d5c7131fa1c71f5f3adefa0305d4d42..a9d068f5941bb7837047e43f595933979cb3819d 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/provenance.txt @@ -1,7 +1,7 @@ ============================================================== PROVENANCE: ultraep__tierbprobe__gbs8 ============================================================== -captured_at : 2026-08-26T14:57:57+00:00 +captured_at : 2026-08-26T15:43:26+00:00 hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 kernel : 6.8.0-1043-nvidia cwd : /workspace/blackwell_run_scripts @@ -21,7 +21,7 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ 1, 120 MHz, 1965 MHz, 1000.00 W, 34 2, 120 MHz, 1965 MHz, 1000.00 W, 34 3, 120 MHz, 1965 MHz, 1000.00 W, 30 - 4, 120 MHz, 1965 MHz, 1000.00 W, 29 + 4, 120 MHz, 1965 MHz, 1000.00 W, 30 5, 120 MHz, 1965 MHz, 1000.00 W, 34 6, 120 MHz, 1965 MHz, 1000.00 W, 34 7, 120 MHz, 1965 MHz, 1000.00 W, 29 @@ -64,7 +64,7 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ UltraEP main 94cab099b44f DeepEP HEAD 34152ae28f80 (+3 untracked: build output) MoonEP megatron-integration c27d35da15f2 - blackwell_run_scripts granularity-ultraep 81efb04918d8 + blackwell_run_scripts granularity-ultraep 05e7365f983f ---- environment --------------------------------------------- CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv index 0812871d3f04ec69febfb8f42e66394d3be53ff6..3ddb77163721b57ab739f9ed51f6ab42e12681ce 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 4 1, 1 2, 1 3, 1 @@ -26,7 +26,7 @@ 1, 4 2, 4 3, 4 -4, 1 +4, 4 5, 1 6, 1 7, 1 @@ -38,14 +38,14 @@ 5, 4 6, 4 7, 4 -0, 761 -1, 725 -2, 737 -3, 725 -4, 725 -5, 725 -6, 747 -7, 725 +0, 821 +1, 821 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 0, 1537 1, 1537 2, 1537 @@ -62,14 +62,14 @@ 5, 2471 6, 2471 7, 2471 -0, 2871 -1, 2871 -2, 2871 -3, 2871 -4, 2871 -5, 2871 -6, 2871 -7, 2871 +0, 2847 +1, 2843 +2, 2851 +3, 2849 +4, 2847 +5, 2867 +6, 2837 +7, 2839 0, 28477 1, 28565 2, 28411 @@ -81,19 +81,19 @@ 0, 34335 1, 34419 2, 34269 -3, 34267 -4, 34267 -5, 34265 +3, 32155 +4, 34203 +5, 34201 6, 34401 7, 34419 -0, 39471 -1, 39555 -2, 39405 -3, 39405 -4, 39405 -5, 39401 -6, 39537 -7, 39555 +0, 39031 +1, 39129 +2, 38987 +3, 38983 +4, 38957 +5, 38979 +6, 39095 +7, 39103 0, 39771 1, 39855 2, 39705 @@ -110,14 +110,6 @@ 5, 39701 6, 39837 7, 39855 -0, 39779 -1, 39863 -2, 39713 -3, 39713 -4, 39713 -5, 39709 -6, 39845 -7, 39863 0, 39781 1, 39865 2, 39715 @@ -126,14 +118,22 @@ 5, 39711 6, 39847 7, 39865 -0, 111665 -1, 111781 -2, 111693 -3, 111607 -4, 111671 -5, 111589 -6, 111749 -7, 111745 +0, 39781 +1, 39865 +2, 39715 +3, 39715 +4, 39715 +5, 39711 +6, 39847 +7, 39865 +0, 111693 +1, 111809 +2, 111725 +3, 111649 +4, 111721 +5, 111673 +6, 111821 +7, 111827 0, 112149 1, 112235 2, 112149 @@ -150,38 +150,30 @@ 5, 112085 6, 112221 7, 112239 -0, 112155 -1, 112239 -2, 112153 -3, 112089 -4, 112153 -5, 112085 -6, 112221 -7, 112239 -0, 114517 -1, 112583 -2, 114539 -3, 114495 -4, 114525 -5, 114443 -6, 112557 -7, 112611 -0, 115425 -1, 113483 -2, 115419 -3, 115389 -4, 115449 -5, 115383 -6, 113435 -7, 113495 -0, 127845 -1, 129971 -2, 127845 -3, 129825 -4, 129889 -5, 129817 -6, 130023 -7, 129971 +0, 114405 +1, 112485 +2, 114387 +3, 114353 +4, 114365 +5, 114335 +6, 112375 +7, 112441 +0, 115241 +1, 113277 +2, 115239 +3, 115175 +4, 115239 +5, 115171 +6, 113259 +7, 113277 +0, 115949 +1, 114023 +2, 115985 +3, 115877 +4, 115889 +5, 115853 +6, 113897 +7, 113995 0, 127845 1, 129971 2, 127845 @@ -206,27 +198,35 @@ 5, 129817 6, 130023 7, 129971 -0, 126899 -1, 129025 -2, 126899 -3, 128879 -4, 128943 -5, 128871 -6, 129077 -7, 129025 -0, 124071 -1, 125963 +0, 127545 +1, 129671 +2, 127545 +3, 129525 +4, 129589 +5, 129517 +6, 129723 +7, 129671 +0, 124391 +1, 126197 2, 124071 3, 126051 -4, 125727 -5, 125723 -6, 126089 -7, 126197 -0, 321 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 68 -7, 124723 +4, 126179 +5, 126301 +6, 126633 +7, 126455 +0, 122003 +1, 124129 +2, 122067 +3, 124693 +4, 125339 +5, 124679 +6, 124827 +7, 124449 +0, 1 +1, 1 +2, 3 +3, 132 +4, 125337 +5, 644 +6, 4 +7, 4 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log index 830b08367c3622c74934514134049aa0196af626..042fe78399366391a8be1bace372e4d6f171b4cc 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log @@ -1,4 +1,4 @@ -[15:02:25] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:48:32] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) ============================================================== DSV2_LITE_L24 / ultraep (slots/rank=8 policy=matched) E=64 K=6 H'=1408 m=8 @@ -7,13 +7,17 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-UltraEP @ eaf829e7 ============================================================== -W0826 15:02:27.046000 54788 torch/distributed/run.py:766] -W0826 15:02:27.046000 54788 torch/distributed/run.py:766] ***************************************** -W0826 15:02:27.046000 54788 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:02:27.046000 54788 torch/distributed/run.py:766] ***************************************** +W0826 15:48:34.068000 140201 torch/distributed/run.py:766] +W0826 15:48:34.068000 140201 torch/distributed/run.py:766] ***************************************** +W0826 15:48:34.068000 140201 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:48:34.068000 140201 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 @@ -579,26 +583,22 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 23 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank 203 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 - - +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to [Gloo] Rank 7 peer ranks. 7Expected number of connected peer ranks is : is connected to 77 - peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 @@ -606,28 +606,28 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to [Gloo] Rank 0 peer ranks. 0Expected number of connected peer ranks is : is connected to 00 - peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank3]:[W826 15:02:45.186808861 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:02:45.186966330 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:02:45.189668353 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:02:45.189719675 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:02:45.189816631 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 15:02:45.189828922 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:02:45.191376132 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 15:48:52.063251487 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:48:52.072814614 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:48:52.072865115 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:48:52.072919292 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:48:52.072936936 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:48:52.072994889 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:48:52.074163235 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.035 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:02:45.355922139 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.382 seconds -time to initialize megatron (seconds): 6.708 -[after megatron is initialized] datetime: 2026-08-26 15:02:50 +[rank0]:[W826 15:48:52.221939488 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.089 seconds +time to initialize megatron (seconds): 7.557 +[after megatron is initialized] datetime: 2026-08-26 15:48:58 building GPT model ... /workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. warnings.warn( @@ -693,37 +693,32 @@ Params for bucket 2 (48503296 elements, 48503296 padded size): module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.23.self_attention.linear_kv_down_proj.weight module.decoder.layers.23.self_attention.linear_proj.weight - module.decoder.layers.23.input_layernorm.weight module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight module.decoder.final_layernorm.weight module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight module.decoder.layers.23.mlp.router.weight module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.input_layernorm.weight module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight Params for bucket 3 (40901632 elements, 40901632 padded size): - module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.22.self_attention.linear_kv_down_proj.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.weight - module.decoder.layers.22.self_attention.linear_q_proj.weight - module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.21.self_attention.linear_kv_down_proj.weight module.decoder.layers.21.self_attention.linear_q_proj.weight - module.decoder.layers.22.mlp.router.weight - module.decoder.layers.22.pre_mlp_layernorm.weight - module.decoder.layers.22.input_layernorm.weight - module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight module.decoder.layers.21.mlp.router.weight module.decoder.layers.21.pre_mlp_layernorm.weight Params for bucket 4 (41163264 elements, 41163264 padded size): module.decoder.layers.21.self_attention.linear_proj.weight - module.decoder.layers.20.mlp.router.weight - module.decoder.layers.20.pre_mlp_layernorm.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.21.input_layernorm.weight module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight @@ -731,22 +726,27 @@ Params for bucket 4 (41163264 elements, 41163264 padded size): module.decoder.layers.20.self_attention.linear_q_proj.weight module.decoder.layers.20.self_attention.linear_proj.weight module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight Params for bucket 5 (42734080 elements, 42734080 padded size): - module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.19.mlp.router.weight - module.decoder.layers.19.self_attention.linear_q_proj.weight - module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight module.decoder.layers.19.self_attention.linear_kv_up_proj.weight module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.input_layernorm.weight module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight Params for bucket 6 (40901632 elements, 40901632 padded size): - module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.18.self_attention.linear_kv_down_proj.weight module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.18.self_attention.linear_q_proj.weight module.decoder.layers.18.self_attention.linear_proj.weight module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight @@ -762,17 +762,17 @@ Params for bucket 6 (40901632 elements, 40901632 padded size): module.decoder.layers.17.pre_mlp_layernorm.weight Params for bucket 7 (41163264 elements, 41163264 padded size): module.decoder.layers.17.self_attention.linear_proj.weight - module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.16.self_attention.linear_kv_down_proj.weight - module.decoder.layers.16.self_attention.linear_q_proj.weight module.decoder.layers.16.self_attention.linear_proj.weight module.decoder.layers.16.input_layernorm.weight - module.decoder.layers.16.mlp.router.weight module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.router.weight module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight Params for bucket 8 (42734080 elements, 42734080 padded size): module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight @@ -781,35 +781,30 @@ Params for bucket 8 (42734080 elements, 42734080 padded size): module.decoder.layers.15.self_attention.linear_kv_down_proj.weight module.decoder.layers.15.self_attention.linear_q_proj.weight module.decoder.layers.15.self_attention.linear_proj.weight - module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.pre_mlp_layernorm.weight module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight module.decoder.layers.15.input_layernorm.weight module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight module.decoder.layers.14.self_attention.linear_kv_down_proj.weight module.decoder.layers.14.self_attention.linear_q_proj.weight module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.13.pre_mlp_layernorm.weight - module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.13.self_attention.linear_kv_down_proj.weight - module.decoder.layers.13.self_attention.linear_q_proj.weight - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.14.input_layernorm.weight - module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight Params for bucket 10 (41163264 elements, 41163264 padded size): module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.12.input_layernorm.weight - module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight module.decoder.layers.13.input_layernorm.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.weight @@ -817,22 +812,33 @@ Params for bucket 10 (41163264 elements, 41163264 padded size): module.decoder.layers.12.self_attention.linear_kv_down_proj.weight module.decoder.layers.12.self_attention.linear_q_proj.weight module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight Params for bucket 11 (42734080 elements, 42734080 padded size): - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.input_layernorm.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.11.self_attention.linear_kv_down_proj.weight module.decoder.layers.11.self_attention.linear_q_proj.weight module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (40901632 elements, 40901632 padded size): - module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.10.self_attention.linear_q_proj.weight module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.10.input_layernorm.weight @@ -840,12 +846,6 @@ Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.9.self_attention.linear_kv_up_proj.weight module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.9.self_attention.linear_kv_down_proj.weight - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.linear_q_proj.weight Params for bucket 13 (41163264 elements, 41163264 padded size): module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight @@ -854,16 +854,16 @@ Params for bucket 13 (41163264 elements, 41163264 padded size): module.decoder.layers.8.self_attention.linear_kv_down_proj.weight module.decoder.layers.8.self_attention.linear_q_proj.weight module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.9.input_layernorm.weight module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.input_layernorm.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight Params for bucket 14 (42734080 elements, 42734080 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.weight module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight module.decoder.layers.7.self_attention.linear_kv_down_proj.weight module.decoder.layers.7.self_attention.linear_q_proj.weight module.decoder.layers.7.self_attention.linear_proj.weight @@ -873,29 +873,23 @@ Params for bucket 14 (42734080 elements, 42734080 padded size): module.decoder.layers.7.mlp.router.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight Params for bucket 15 (40901632 elements, 40901632 padded size): - module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.6.self_attention.linear_kv_down_proj.weight module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight module.decoder.layers.6.self_attention.linear_proj.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.linear_q_proj.weight module.decoder.layers.6.input_layernorm.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.5.self_attention.linear_kv_down_proj.weight module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight Params for bucket 16 (41163264 elements, 41163264 padded size): - module.decoder.layers.5.input_layernorm.weight - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.input_layernorm.weight - module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.weight @@ -903,49 +897,55 @@ Params for bucket 16 (41163264 elements, 41163264 padded size): module.decoder.layers.4.self_attention.linear_kv_down_proj.weight module.decoder.layers.4.self_attention.linear_q_proj.weight module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight Params for bucket 17 (42734080 elements, 42734080 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.3.self_attention.linear_kv_down_proj.weight module.decoder.layers.3.self_attention.linear_q_proj.weight module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.3.input_layernorm.weight module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.2.self_attention.linear_kv_down_proj.weight module.decoder.layers.2.self_attention.linear_kv_up_proj.weight module.decoder.layers.2.self_attention.linear_q_proj.weight module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.1.self_attention.linear_kv_down_proj.weight - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.input_layernorm.weight - module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.weight - module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.1.self_attention.linear_q_proj.weight Params for bucket 19 (71436288 elements, 71436288 padded size): module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.linear_fc2.weight - module.decoder.layers.1.input_layernorm.weight module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight Params for bucket 20 (116789760 elements, 116789760 padded size): module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight - module.decoder.layers.0.self_attention.linear_q_proj.weight - module.embedding.word_embeddings.weight - module.decoder.layers.0.self_attention.linear_kv_down_proj.weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight module.decoder.layers.0.self_attention.linear_proj.weight module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 Params for bucket 1 (40370176 elements, 40370176 padded size): module.decoder.layers.23.mlp.experts.linear_fc2.weight7 @@ -961,33 +961,33 @@ Params for bucket 1 (40370176 elements, 40370176 padded size): module.decoder.layers.23.mlp.experts.linear_fc1.weight5 Params for bucket 2 (40370176 elements, 40370176 padded size): module.decoder.layers.23.mlp.experts.linear_fc1.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 - module.decoder.layers.22.mlp.experts.linear_fc2.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight5 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 - module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 - module.decoder.layers.22.mlp.experts.linear_fc1.weight5 Params for bucket 4 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 module.decoder.layers.21.mlp.experts.linear_fc2.weight0 - module.decoder.layers.22.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 module.decoder.layers.21.mlp.experts.linear_fc2.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 @@ -1015,36 +1015,36 @@ Params for bucket 7 (40370176 elements, 40370176 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 module.decoder.layers.20.mlp.experts.linear_fc1.weight0 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 module.decoder.layers.20.mlp.experts.linear_fc1.weight5 module.decoder.layers.20.mlp.experts.linear_fc1.weight1 module.decoder.layers.20.mlp.experts.linear_fc1.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 Params for bucket 8 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight1 - module.decoder.layers.19.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight2 module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 module.decoder.layers.19.mlp.experts.linear_fc2.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 module.decoder.layers.18.mlp.experts.linear_fc2.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 Params for bucket 10 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc1.weight4 module.decoder.layers.18.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight1 @@ -1053,8 +1053,8 @@ Params for bucket 10 (40370176 elements, 40370176 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight2 Params for bucket 11 (40370176 elements, 40370176 padded size): module.decoder.layers.17.mlp.experts.linear_fc2.weight5 - module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight0 @@ -1065,16 +1065,16 @@ Params for bucket 11 (40370176 elements, 40370176 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight7 Params for bucket 12 (40370176 elements, 40370176 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight6 - module.decoder.layers.17.mlp.experts.linear_fc1.weight3 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 module.decoder.layers.17.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc1.weight5 module.decoder.layers.17.mlp.experts.linear_fc1.weight2 module.decoder.layers.17.mlp.experts.linear_fc1.weight1 Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight7 - module.decoder.layers.16.mlp.experts.linear_fc2.weight3 module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 module.decoder.layers.16.mlp.experts.linear_fc2.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight6 @@ -1085,8 +1085,8 @@ Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc1.weight5 Params for bucket 14 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc1.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc2.weight5 module.decoder.layers.16.mlp.experts.linear_fc1.weight1 @@ -1095,8 +1095,8 @@ Params for bucket 14 (40370176 elements, 40370176 padded size): module.decoder.layers.15.mlp.experts.linear_fc2.weight4 Params for bucket 15 (40370176 elements, 40370176 padded size): module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 module.decoder.layers.15.mlp.experts.linear_fc1.weight3 @@ -1110,117 +1110,117 @@ Params for bucket 16 (40370176 elements, 40370176 padded size): module.decoder.layers.14.mlp.experts.linear_fc2.weight3 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight1 module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 module.decoder.layers.14.mlp.experts.linear_fc2.weight4 module.decoder.layers.14.mlp.experts.linear_fc2.weight2 module.decoder.layers.14.mlp.experts.linear_fc2.weight1 Params for bucket 17 (40370176 elements, 40370176 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight4 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 module.decoder.layers.14.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 module.decoder.layers.14.mlp.experts.linear_fc1.weight2 module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 Params for bucket 18 (40370176 elements, 40370176 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight5 - module.decoder.layers.13.mlp.experts.linear_fc2.weight7 - module.decoder.layers.13.mlp.experts.linear_fc2.weight3 - module.decoder.layers.13.mlp.experts.linear_fc2.weight2 - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 module.decoder.layers.13.mlp.experts.linear_fc1.weight7 Params for bucket 19 (40370176 elements, 40370176 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc1.weight5 module.decoder.layers.13.mlp.experts.linear_fc1.weight2 module.decoder.layers.13.mlp.experts.linear_fc1.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 module.decoder.layers.12.mlp.experts.linear_fc2.weight5 module.decoder.layers.12.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc1.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight5 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc1.weight6 - module.decoder.layers.12.mlp.experts.linear_fc1.weight4 Params for bucket 21 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc2.weight5 - module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 Params for bucket 22 (40370176 elements, 40370176 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight6 - module.decoder.layers.10.mlp.experts.linear_fc2.weight4 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 Params for bucket 26 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight4 Params for bucket 27 (40370176 elements, 40370176 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 @@ -1228,8 +1228,8 @@ Params for bucket 27 (40370176 elements, 40370176 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 28 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 @@ -1240,12 +1240,12 @@ Params for bucket 28 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 29 (40370176 elements, 40370176 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 30 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight0 @@ -1255,13 +1255,13 @@ Params for bucket 30 (40370176 elements, 40370176 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 31 (40370176 elements, 40370176 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 @@ -1270,94 +1270,94 @@ Params for bucket 31 (40370176 elements, 40370176 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight6 Params for bucket 32 (40370176 elements, 40370176 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 Params for bucket 34 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 Params for bucket 35 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 Params for bucket 36 (40370176 elements, 40370176 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 38 (40370176 elements, 40370176 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 Params for bucket 39 (40370176 elements, 40370176 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 Params for bucket 40 (17301504 elements, 17301504 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:02:54 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:49:01 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1365,7 +1365,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1378,14 +1378,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:02:54 +[after dataloaders are built] datetime: 2026-08-26 15:49:02 done with setup ... -training ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3598.16, 3604.40) - train/valid/test-data-iterators-setup ..........: (71.76, 87.61) + model-and-optimizer-setup ......................: (3692.35, 3701.73) + train/valid/test-data-iterators-setup ..........: (71.73, 85.06) +training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 15:02:54 +[before the start of training step] datetime: 2026-08-26 15:49:02 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1404,12 +1404,12 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1444,10 +1444,6 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1456,6 +1452,10 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1500,934 +1500,934 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=47653 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49446 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48667 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=47653 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49446 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=47653 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48667 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49446 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48667 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48420 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=47653 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48420 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48558 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48558 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48018 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48558 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48018 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48558 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48121 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48121 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48018 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49529 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48018 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49529 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49529 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49529 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48519 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=47476 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48519 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=47476 actual_valid_recv=48000 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=47476 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=47476 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48519 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48519 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=46521 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49469 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49544 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=46521 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49469 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49544 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49469 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49544 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=47882 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49544 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=47251 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=47882 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=47251 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=47251 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=47103 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=47103 actual_valid_recv=47616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=47103 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=47103 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47521 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47831 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47521 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47831 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47831 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48164 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=47831 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48164 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48551 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48450 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=47882 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48551 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48450 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48382 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=47882 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48382 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48164 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48382 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48164 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48787 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=47127 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48668 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=47127 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48787 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48668 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48668 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=47127 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48668 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=47127 actual_valid_recv=48000 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=46684 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=46684 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=47266 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48085 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=47266 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48085 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48085 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48085 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=47266 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48551 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=47266 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48551 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49373 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48225 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49373 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=45979 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=48765 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48225 actual_valid_recv=48640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=45979 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=48765 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=45979 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=48765 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48787 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=45979 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48787 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=46872 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=46872 actual_valid_recv=47488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=46872 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=46872 actual_valid_recv=47488 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47986 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47986 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47986 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47986 actual_valid_recv=48512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49233 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=47552 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=47552 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49233 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=47552 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48362 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48362 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48225 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48225 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48362 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48392 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48392 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=47805 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48362 actual_valid_recv=49024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48392 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=47805 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=47805 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49489 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49489 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49489 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47735 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47735 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49489 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47735 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47777 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47777 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48645 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=47235 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48645 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=47235 actual_valid_recv=47744 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48645 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49118 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48645 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=47235 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=46883 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49118 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48469 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=47235 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=46883 actual_valid_recv=47360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48469 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47459 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=46883 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47459 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=46883 actual_valid_recv=47360 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47617 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47777 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47617 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=47777 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=46765 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48587 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=46765 actual_valid_recv=47360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48587 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48587 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=46765 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48353 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=46765 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48353 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48353 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48353 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47894 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47894 actual_valid_recv=48512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47617 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=47617 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=47734 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=47734 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47894 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=47894 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=47734 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=47734 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=23 total_expected_recv=1133511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267520 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=23 total_expected_recv=1134190 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265600 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=23 total_expected_recv=1125260 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=23 total_expected_recv=1122397 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=277120 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=23 total_expected_recv=1132282 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=23 total_expected_recv=1134190 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265600 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=23 total_expected_recv=1125260 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=23 total_expected_recv=1133313 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267264 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=23 total_expected_recv=1133511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267520 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=23 total_expected_recv=1122397 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=277120 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=23 total_expected_recv=1134581 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265472 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=23 total_expected_recv=1132282 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=23 total_expected_recv=1128434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:23] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 29181.5 | throughput per GPU (TFLOP/s/GPU): 4.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092127E+01 | loss scale: 1.0 | grad norm: 30.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | + [2026-08-26 15:49:30] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 28176.2 | throughput per GPU (TFLOP/s/GPU): 4.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092127E+01 | loss scale: 1.0 | grad norm: 30.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | Number of parameters in transformer block in billions: 13.53 Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 13.74 @@ -2436,18 +2436,14 @@ Theoretical memory footprints: weight and optimizer=98244.96 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 37110.0849609375 | max allocated: 104505.06396484375 | reserved: 107726.0 | max reserved: 107726.0 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -2460,6 +2456,10 @@ Theoretical memory footprints: weight and optimizer=98244.96 MB warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -2482,10243 +2482,10243 @@ Theoretical memory footprints: weight and optimizer=98244.96 MB warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=48091 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=61440 expected_recv=47725 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=48091 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=61440 expected_recv=47725 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47977 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47977 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47820 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47977 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47820 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=47977 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49395 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=47761 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48763 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=47761 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48763 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48763 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=47761 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=47761 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48451 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48965 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48451 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48965 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48697 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48451 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48529 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48697 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48965 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48697 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48451 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48529 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48697 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48965 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=47048 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=47048 actual_valid_recv=47616 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48708 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=47048 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48708 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=47048 actual_valid_recv=47616 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=47292 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48708 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48060 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48060 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=47292 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=47292 actual_valid_recv=47616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48060 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48060 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47967 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47804 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47967 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47804 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47967 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=47967 actual_valid_recv=48512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48522 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48762 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48762 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48522 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=47712 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48522 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48762 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48762 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=47712 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=47712 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49504 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47860 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=47712 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49504 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47860 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47632 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47632 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49504 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49530 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49504 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49530 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47615 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47632 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47860 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47615 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47850 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47632 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=47860 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47850 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47850 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=47005 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47850 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49530 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=47005 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47615 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49530 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=47615 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=48346 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48877 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48200 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=48346 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48877 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=48346 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48200 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=48346 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=47005 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48877 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=47005 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48877 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48500 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48500 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48055 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=47920 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48055 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48500 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=47920 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48200 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48200 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48456 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=46896 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48456 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=46896 actual_valid_recv=47744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48055 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48055 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=46896 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48456 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=46896 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48456 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=47125 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=47125 actual_valid_recv=47744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=47125 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48405 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=46948 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48405 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48405 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=46948 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48405 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47820 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47820 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=46948 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=46948 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47851 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=47116 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47851 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=47116 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=47851 actual_valid_recv=48512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49112 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49112 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49112 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=47116 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49112 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=47116 actual_valid_recv=47872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48769 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48769 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48851 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48769 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=46024 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=46719 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=46024 actual_valid_recv=46592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=46719 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48851 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=46719 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=46719 actual_valid_recv=47360 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47751 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47751 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47601 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=46024 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47601 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=46024 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47751 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=46982 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=47751 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=46982 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47561 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49497 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48369 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49497 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47938 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48369 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47561 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47938 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47561 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48369 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47938 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=47938 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49497 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49497 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=23 total_expected_recv=1128437 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=23 total_expected_recv=1136387 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263808 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=23 total_expected_recv=1129195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=23 total_expected_recv=1123546 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=276096 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=23 total_expected_recv=1134273 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=266624 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=23 total_expected_recv=1128391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=23 total_expected_recv=1134221 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267008 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=23 total_expected_recv=1129195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=23 total_expected_recv=1134273 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=266624 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=23 total_expected_recv=1136387 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263808 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=23 total_expected_recv=1129518 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:24] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 642.7 | throughput per GPU (TFLOP/s/GPU): 198.5 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091995E+01 | loss scale: 1.0 | grad norm: 30.567 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=23 total_expected_recv=1128437 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=23 total_expected_recv=1128391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:49:30] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 641.2 | throughput per GPU (TFLOP/s/GPU): 199.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091995E+01 | loss scale: 1.0 | grad norm: 30.567 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48263 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=47905 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48263 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48263 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=47905 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=47135 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=47135 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48217 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=47135 actual_valid_recv=47616 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48217 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48217 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48686 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48686 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48651 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=47656 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48651 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48651 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=47656 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48686 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48651 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=47656 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48686 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48531 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=47875 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48531 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=47875 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48589 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48531 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48589 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48589 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=47875 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48589 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=47875 actual_valid_recv=48640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47592 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47592 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47592 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47878 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47592 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47878 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=47878 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49526 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47867 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47867 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47867 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47484 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47867 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47484 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47484 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=47484 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48019 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=47333 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48019 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48019 actual_valid_recv=48768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=47333 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=47049 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=47049 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=47049 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48303 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49472 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48303 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49472 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49472 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48201 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48201 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=47323 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=47323 actual_valid_recv=48128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=47323 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=47323 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=47110 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=47110 actual_valid_recv=47744 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48241 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48241 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=47110 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48241 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=47110 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48241 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=46290 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=46290 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=46290 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=46290 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=46959 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=46959 actual_valid_recv=47616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48392 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=46959 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=46959 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48392 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48392 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49580 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49580 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=46562 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49639 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=46562 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49639 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49639 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49639 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=46562 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=45713 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=46562 actual_valid_recv=47104 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=45713 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=45713 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48362 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=45713 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48362 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48756 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48756 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=46991 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=46991 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=46991 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=46991 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=46596 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=46596 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48756 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48756 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=46596 actual_valid_recv=47488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47358 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47358 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46848 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47993 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47993 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47993 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47993 actual_valid_recv=49024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47358 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=47358 actual_valid_recv=47872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=23 total_expected_recv=1135446 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264320 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=23 total_expected_recv=1132664 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265984 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=23 total_expected_recv=1123780 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275200 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=23 total_expected_recv=1130427 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=23 total_expected_recv=1134893 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263680 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=23 total_expected_recv=1128698 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270080 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=23 total_expected_recv=1126154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=23 total_expected_recv=1134893 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263680 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=23 total_expected_recv=1135446 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264320 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=23 total_expected_recv=1130427 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=23 total_expected_recv=1131906 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267136 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:24] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 636.8 | throughput per GPU (TFLOP/s/GPU): 200.4 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952354E+00 | loss scale: 1.0 | grad norm: 42.232 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=47005 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=23 total_expected_recv=1132664 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265984 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=23 total_expected_recv=1128698 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270080 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:49:31] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 636.5 | throughput per GPU (TFLOP/s/GPU): 200.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952354E+00 | loss scale: 1.0 | grad norm: 42.232 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=47005 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48777 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=47005 actual_valid_recv=47744 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49215 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=47005 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48777 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48777 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48777 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=46897 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=46897 actual_valid_recv=47488 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47626 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47725 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47626 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47725 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47626 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47725 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=47626 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=45832 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=45832 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=45832 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=45832 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49521 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49521 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=61440 expected_recv=46144 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=61440 expected_recv=46144 actual_valid_recv=46848 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48110 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48110 actual_valid_recv=48640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=47241 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=47241 actual_valid_recv=47616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48110 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48110 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=47562 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=47562 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=47562 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48153 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48153 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48153 actual_valid_recv=48640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48153 actual_valid_recv=48640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48176 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48176 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48176 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=47177 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=47177 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=47177 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=47177 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=46417 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=46417 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=46417 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=46417 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=47588 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=47588 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48142 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48142 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48142 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48142 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=45952 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=45952 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=45952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=45952 slack_slots=15488 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48465 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48465 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=46885 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48465 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=46885 actual_valid_recv=47360 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48465 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=46885 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=46885 actual_valid_recv=47360 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=23 total_expected_recv=1131921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268800 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=23 total_expected_recv=1129377 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=23 total_expected_recv=1131525 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=23 total_expected_recv=1131921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268800 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=23 total_expected_recv=1132497 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269952 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=23 total_expected_recv=1127327 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275712 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=23 total_expected_recv=1133045 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49646 required_capacity=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=23 total_expected_recv=1131525 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=23 total_expected_recv=1120395 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=280832 peak_demand=49646 required_capacity=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=23 total_expected_recv=1127327 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275712 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=23 total_expected_recv=1137881 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263936 peak_demand=49646 required_capacity=1.010050 - [2026-08-26 15:03:25] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 640.1 | throughput per GPU (TFLOP/s/GPU): 199.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477323E+00 | loss scale: 1.0 | grad norm: 7.143 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49498 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] + [2026-08-26 15:49:32] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 639.9 | throughput per GPU (TFLOP/s/GPU): 199.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477323E+00 | loss scale: 1.0 | grad norm: 7.143 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=47069 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49498 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49498 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=47069 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48429 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=47069 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49498 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=47069 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48429 actual_valid_recv=48768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48429 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47721 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47631 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47721 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47721 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47631 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=47721 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=61440 expected_recv=46461 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=61440 expected_recv=46461 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48530 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48530 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48530 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48530 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=61440 expected_recv=47269 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=61440 expected_recv=47269 actual_valid_recv=47872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=61440 expected_recv=46993 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48359 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=61440 expected_recv=46993 actual_valid_recv=47488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48359 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48359 actual_valid_recv=49024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48359 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47616 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47738 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47616 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47616 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47738 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=47616 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49536 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49536 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=45818 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49536 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=45818 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=47035 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=45818 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49536 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=45818 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=47035 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48316 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48316 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=47035 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=47035 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48316 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48316 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47367 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47985 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47367 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47985 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47985 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=46554 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=46554 actual_valid_recv=47232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48798 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=47985 actual_valid_recv=48640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48798 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=46554 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=46554 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48798 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48798 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47428 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47428 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47925 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47925 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47428 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47925 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47925 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=47428 actual_valid_recv=48128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48377 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48377 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=47343 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=47343 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=47343 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=46055 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=46055 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=46055 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=46055 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=46713 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=46713 actual_valid_recv=47232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48640 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=46713 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=46713 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48640 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48640 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48442 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48442 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=46911 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=46911 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=23 total_expected_recv=1139534 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=260864 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=23 total_expected_recv=1122666 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=276480 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=23 total_expected_recv=1127280 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=23 total_expected_recv=1136609 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=262784 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=23 total_expected_recv=1123786 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275328 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=23 total_expected_recv=1122043 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=278272 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=23 total_expected_recv=1127280 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=23 total_expected_recv=1140820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=259584 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=23 total_expected_recv=1123786 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275328 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=23 total_expected_recv=1131230 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268160 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:26] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 637.8 | throughput per GPU (TFLOP/s/GPU): 200.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119604E+01 | loss scale: 1.0 | grad norm: 396.977 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47646 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=23 total_expected_recv=1122666 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=276480 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=23 total_expected_recv=1140820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=259584 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 15:49:32] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 636.0 | throughput per GPU (TFLOP/s/GPU): 200.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119604E+01 | loss scale: 1.0 | grad norm: 396.977 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47706 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47646 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47706 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47706 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47646 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47706 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=47646 actual_valid_recv=48128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=61440 expected_recv=46853 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=61440 expected_recv=46853 actual_valid_recv=47616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48499 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48499 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48192 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=47476 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48192 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=47476 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49326 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47713 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47713 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=47476 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47943 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=47476 actual_valid_recv=48128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47943 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47713 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47713 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47943 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=47943 actual_valid_recv=48512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=46151 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=46151 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=46151 actual_valid_recv=46720 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49589 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49589 actual_valid_recv=50688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50688 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=45763 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=45763 actual_valid_recv=46720 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49589 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49589 actual_valid_recv=50688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49605 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50688 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=45746 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49605 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=45746 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=45746 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=45746 actual_valid_recv=46208 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48075 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48075 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49605 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49605 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=47277 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=47277 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48075 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=45876 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48075 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=45876 actual_valid_recv=46592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=45876 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=45876 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49475 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49475 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50688 slack_slots=10752 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50688 slack_slots=10752 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46720 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=45807 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49546 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=45807 actual_valid_recv=46720 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46720 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=46081 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=46081 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=46081 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=46081 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47690 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47690 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47662 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47662 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47662 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47690 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=47690 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47967 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47387 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47967 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47967 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47387 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47387 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=47387 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=46414 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=46414 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=47123 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=47123 actual_valid_recv=47616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48233 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48233 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=47123 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=47123 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=46176 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49641 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=46176 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49641 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=23 total_expected_recv=1133970 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264192 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=23 total_expected_recv=1133392 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265856 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=23 total_expected_recv=1135772 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264576 peak_demand=49646 required_capacity=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=23 total_expected_recv=1124312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274432 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=23 total_expected_recv=1116463 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=281472 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=23 total_expected_recv=1136336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=262400 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=23 total_expected_recv=1124312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274432 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=23 total_expected_recv=1123931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275072 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=23 total_expected_recv=1116463 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=281472 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=23 total_expected_recv=1135772 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264576 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=23 total_expected_recv=1139792 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=260352 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:26] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 642.1 | throughput per GPU (TFLOP/s/GPU): 198.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.367741E+00 | loss scale: 1.0 | grad norm: 5.090 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=23 total_expected_recv=1133970 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264192 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 15:49:33] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 642.2 | throughput per GPU (TFLOP/s/GPU): 198.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.367741E+00 | loss scale: 1.0 | grad norm: 5.090 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=47434 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48655 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=47434 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=47434 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48655 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=47434 actual_valid_recv=47872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49621 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=45949 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=45949 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=45949 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49621 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49426 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=45949 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49621 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49426 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49621 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48496 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48496 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=46856 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48496 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=46856 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=46856 actual_valid_recv=47232 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=46856 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47654 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47698 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47654 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47654 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47698 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=47654 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48202 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48202 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=47149 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48202 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=47149 actual_valid_recv=47872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48202 actual_valid_recv=49024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48785 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48785 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=46566 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=46566 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48785 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=46566 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48785 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47753 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47753 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47601 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47601 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47753 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=47753 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47491 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47863 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47491 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47491 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47863 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47491 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47863 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=47863 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=45963 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=45963 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=46448 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=46448 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=46448 actual_valid_recv=47232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=46448 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=46856 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=46856 actual_valid_recv=47744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48496 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48496 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=46856 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48496 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=46856 actual_valid_recv=47744 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48496 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=46410 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=46410 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=46357 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=46410 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=46410 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=46357 actual_valid_recv=46976 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48994 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48994 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=46357 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=46357 actual_valid_recv=46976 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48687 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=46664 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=46664 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48687 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=46664 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=46664 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48371 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48371 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=46977 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48371 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=46977 actual_valid_recv=47360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48371 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=45706 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=45706 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48866 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48866 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48866 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48866 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=46484 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=46484 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=46980 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=48375 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=46980 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=48375 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=46980 actual_valid_recv=47616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=48375 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=23 total_expected_recv=1128842 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=23 total_expected_recv=1133099 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267776 peak_demand=49646 required_capacity=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=23 total_expected_recv=1136167 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264192 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=23 total_expected_recv=1127840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=23 total_expected_recv=1125408 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274816 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=23 total_expected_recv=1136554 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=262016 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=23 total_expected_recv=1128842 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=23 total_expected_recv=1120414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=277888 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=23 total_expected_recv=1136167 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264192 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=23 total_expected_recv=1127840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=23 total_expected_recv=1135644 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263424 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:27] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 641.4 | throughput per GPU (TFLOP/s/GPU): 198.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.973585E+00 | loss scale: 1.0 | grad norm: 3.690 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=47787 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=23 total_expected_recv=1125408 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274816 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 15:49:34] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 641.0 | throughput per GPU (TFLOP/s/GPU): 199.0 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.973585E+00 | loss scale: 1.0 | grad norm: 3.690 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=47787 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48198 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=47787 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=47787 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48198 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=47354 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=47354 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48741 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48741 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=47354 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48741 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48887 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=47354 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=47341 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48887 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=47341 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48768 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48768 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48768 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48768 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=47341 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48887 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=47341 actual_valid_recv=48000 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48887 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47430 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47922 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47430 actual_valid_recv=48128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47922 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47430 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47430 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47922 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=47922 actual_valid_recv=48512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=45710 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=45710 actual_valid_recv=46848 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=47567 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=47567 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48973 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48456 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48973 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48456 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48456 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48276 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48456 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48276 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=47567 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=47079 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=47567 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=47079 actual_valid_recv=47616 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=47079 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=47079 actual_valid_recv=47616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48276 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48276 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49641 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49641 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49647 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49647 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49647 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49641 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49647 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49641 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=45711 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=45711 actual_valid_recv=46592 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48281 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=45711 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=47071 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48281 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=47071 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=45711 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=47071 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=47071 actual_valid_recv=47616 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48281 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48281 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=47199 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48155 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48155 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=47199 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=47199 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48155 actual_valid_recv=48896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48158 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48158 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=45710 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=45710 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48158 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=47154 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48158 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=47154 actual_valid_recv=47488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48199 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=47194 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48199 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=47194 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=47154 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=47154 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48199 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48199 actual_valid_recv=48640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=46298 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=46298 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=46298 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49588 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=46298 actual_valid_recv=46976 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49588 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46976 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=45762 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=45762 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49588 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=45762 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=45762 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49588 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48698 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48698 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=47223 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49392 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=47223 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49392 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=46908 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=46908 actual_valid_recv=47232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48442 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49392 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48442 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48698 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49392 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=46908 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48698 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=46908 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48442 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=47223 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48442 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=47223 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=23 total_expected_recv=1125840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275328 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=23 total_expected_recv=1130625 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269952 peak_demand=49646 required_capacity=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=23 total_expected_recv=1135536 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264832 peak_demand=49646 required_capacity=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=23 total_expected_recv=1123706 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=277120 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=23 total_expected_recv=1134084 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265088 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=23 total_expected_recv=1125840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275328 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=23 total_expected_recv=1133176 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267008 peak_demand=49647 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=23 total_expected_recv=1123706 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=277120 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=23 total_expected_recv=1135536 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264832 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=23 total_expected_recv=1133689 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=23 total_expected_recv=1134084 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265088 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=23 total_expected_recv=1127312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273408 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:28] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 639.3 | throughput per GPU (TFLOP/s/GPU): 199.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.787342E+00 | loss scale: 1.0 | grad norm: 5.890 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | + [2026-08-26 15:49:34] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 639.4 | throughput per GPU (TFLOP/s/GPU): 199.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.787342E+00 | loss scale: 1.0 | grad norm: 5.890 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48474 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48143 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48379 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48474 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48474 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48379 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48143 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48474 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48379 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48379 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48741 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48741 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=46610 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=46610 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=46610 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=46610 actual_valid_recv=47104 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48310 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48310 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=47479 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=47479 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48430 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48073 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48430 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48430 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48073 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48493 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48073 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48430 actual_valid_recv=49024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48493 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48493 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=45861 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=45861 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=45861 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=45861 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=47106 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=47106 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=47106 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48246 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=47106 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48246 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48246 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48353 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48353 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=46998 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=46998 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=47616 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46080 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=45711 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=45711 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=45711 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47508 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=46818 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=46818 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47508 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47508 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=46818 actual_valid_recv=47488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47508 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48535 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47844 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48535 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47844 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47844 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=47844 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=23 total_expected_recv=1128939 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=23 total_expected_recv=1134425 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=23 total_expected_recv=1128332 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271872 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=23 total_expected_recv=1134077 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=267008 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=23 total_expected_recv=1125663 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275584 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=23 total_expected_recv=1132966 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=266240 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=23 total_expected_recv=1121690 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=278784 peak_demand=49646 required_capacity=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=23 total_expected_recv=1134425 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268416 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=23 total_expected_recv=1125663 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=275584 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=23 total_expected_recv=1128332 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271872 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=23 total_expected_recv=1137876 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263296 peak_demand=49646 required_capacity=1.010050 - [2026-08-26 15:03:28] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 645.8 | throughput per GPU (TFLOP/s/GPU): 197.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.612353E+00 | loss scale: 1.0 | grad norm: 3.038 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=23 total_expected_recv=1121690 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=278784 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 15:49:35] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 639.8 | throughput per GPU (TFLOP/s/GPU): 199.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.612353E+00 | loss scale: 1.0 | grad norm: 3.038 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48705 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48459 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=47832 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48459 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48705 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48459 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=47832 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48459 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=47832 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48620 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=47652 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48620 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48724 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=47652 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48724 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48620 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48724 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48724 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=48620 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48324 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48324 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=47554 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=47554 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=47554 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=47554 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48460 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48460 actual_valid_recv=49024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48467 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48069 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48467 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48069 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48467 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48069 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48069 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=46417 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=46417 actual_valid_recv=47232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48131 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=61440 expected_recv=47776 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48131 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=61440 expected_recv=47776 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48131 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48131 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=46977 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=46977 actual_valid_recv=47360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=46977 actual_valid_recv=47360 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48376 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=47360 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48376 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47642 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47642 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47642 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47710 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47710 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=47710 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=61440 expected_recv=47311 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=61440 expected_recv=47311 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48041 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48041 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48391 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48041 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48391 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48394 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48212 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48212 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48394 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48394 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48212 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48394 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=46300 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=46300 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=46300 actual_valid_recv=46848 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=46236 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=46236 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=46236 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=46236 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48510 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48510 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=47179 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=47179 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=47179 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48510 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=47179 actual_valid_recv=47744 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48510 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46080 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=45706 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=45706 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=46443 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=46443 actual_valid_recv=47104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48908 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=46443 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=46443 actual_valid_recv=47104 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=47104 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47415 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47415 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47938 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47415 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47938 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47415 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47938 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=47938 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=23 total_expected_recv=1129820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49646 required_capacity=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=23 total_expected_recv=1131823 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268160 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=23 total_expected_recv=1127932 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=23 total_expected_recv=1122831 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=276608 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=23 total_expected_recv=1135130 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264064 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=23 total_expected_recv=1127684 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272640 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=23 total_expected_recv=1127932 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=23 total_expected_recv=1134577 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265088 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=23 total_expected_recv=1129820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=23 total_expected_recv=1131823 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268160 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=23 total_expected_recv=1134171 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=265088 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:29] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 640.7 | throughput per GPU (TFLOP/s/GPU): 199.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.076241E+00 | loss scale: 1.0 | grad norm: 29.812 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | + [2026-08-26 15:49:36] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 638.8 | throughput per GPU (TFLOP/s/GPU): 199.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.076241E+00 | loss scale: 1.0 | grad norm: 29.812 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=47764 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=47764 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48306 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=47764 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48306 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=47764 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48306 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48666 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48666 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=47614 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48666 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=47614 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48714 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=47614 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48714 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=47614 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47698 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48714 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47698 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48714 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47654 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47698 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47654 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47654 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=47654 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=47848 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49611 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=47848 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49611 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49611 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48720 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48461 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48720 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49611 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48461 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48461 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=47632 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=47632 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48720 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48720 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48213 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48213 actual_valid_recv=48896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=47632 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48213 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=47632 actual_valid_recv=48256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48135 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48135 actual_valid_recv=48640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48146 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48146 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48715 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48146 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48135 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48715 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48135 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48715 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47724 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47628 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47724 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47628 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47628 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=47628 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47364 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47988 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47364 actual_valid_recv=47872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47988 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47872 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=47988 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48636 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=46718 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48636 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=46718 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=46718 actual_valid_recv=47232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47782 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47782 actual_valid_recv=48512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47571 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47571 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47571 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=45804 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47782 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49547 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=45804 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=47782 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49547 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=45804 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=45804 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=46774 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=46774 actual_valid_recv=47744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49547 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49547 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=46221 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=46221 actual_valid_recv=46848 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49130 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49130 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=46221 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=46221 actual_valid_recv=46848 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47700 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47700 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47700 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47700 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47651 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47442 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47651 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47910 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47651 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47442 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47910 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=47651 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47910 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=47910 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47553 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47553 actual_valid_recv=48384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47796 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47796 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47796 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=47796 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=23 total_expected_recv=1127801 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=23 total_expected_recv=1130291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49644 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=23 total_expected_recv=1128868 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=23 total_expected_recv=1130140 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=23 total_expected_recv=1131367 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268160 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=23 total_expected_recv=1136935 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=263168 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=23 total_expected_recv=1127801 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=23 total_expected_recv=1130291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=23 total_expected_recv=1129916 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269056 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=23 total_expected_recv=1128650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=23 total_expected_recv=1130140 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49645 required_capacity=1.010030 - [2026-08-26 15:03:29] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 644.8 | throughput per GPU (TFLOP/s/GPU): 197.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.639501E+00 | loss scale: 1.0 | grad norm: 64.594 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=23 total_expected_recv=1129916 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269056 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 15:49:36] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 639.6 | throughput per GPU (TFLOP/s/GPU): 199.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.639501E+00 | loss scale: 1.0 | grad norm: 64.594 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48125 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48695 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48853 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48125 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48853 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48695 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48695 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49549 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47814 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49549 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47633 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47814 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47633 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47814 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47633 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47633 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49549 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=47814 actual_valid_recv=48512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49549 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48449 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=46904 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48449 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48449 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=46904 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=46904 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48449 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=47193 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=47193 actual_valid_recv=47744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48159 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=47193 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48159 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=47193 actual_valid_recv=47744 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48491 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48665 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=47840 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48491 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48665 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=47840 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48665 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=47840 actual_valid_recv=48640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=48640 slack_slots=12800 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=46691 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48853 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=46691 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49452 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=46691 actual_valid_recv=47488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48853 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49452 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=47488 slack_slots=13952 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47665 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47687 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47665 actual_valid_recv=48256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47665 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47687 actual_valid_recv=48128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=47665 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48128 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=48256 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47965 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47882 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47965 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47882 actual_valid_recv=48512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47965 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47882 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47965 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=47882 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=48512 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49381 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=61440 expected_recv=46579 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=61440 expected_recv=46579 actual_valid_recv=47232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=47232 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49381 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48043 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=47497 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48043 actual_valid_recv=48768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=47497 actual_valid_recv=48000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48768 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=61440 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=47497 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=61440 expected_recv=45706 actual_valid_recv=46592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=47497 actual_valid_recv=48000 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48523 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=48000 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=46829 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48523 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48523 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=46829 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=46829 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=46829 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=47744 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47705 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47705 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47705 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47705 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=47646 actual_valid_recv=48384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=48384 slack_slots=13056 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=45709 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=46208 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=46365 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=46365 actual_valid_recv=46848 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=46848 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50560 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=46592 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=46464 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50432 slack_slots=11008 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=46336 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=23 total_expected_recv=1132031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268288 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=23 total_expected_recv=1135171 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264064 peak_demand=49646 required_capacity=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=23 total_expected_recv=1130941 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268544 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=23 total_expected_recv=1124233 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274304 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=23 total_expected_recv=1134243 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264832 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=23 total_expected_recv=1131026 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269056 peak_demand=49645 required_capacity=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=23 total_expected_recv=1135171 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=264064 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=23 total_expected_recv=1124233 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274304 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=23 total_expected_recv=1127543 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49645 required_capacity=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=23 total_expected_recv=1128780 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49646 required_capacity=1.010050 - [2026-08-26 15:03:30] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 639.8 | throughput per GPU (TFLOP/s/GPU): 199.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.624613E+00 | loss scale: 1.0 | grad norm: 71.371 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | -[after training is done] datetime: 2026-08-26 15:03:30 -[rank0]:[W826 15:03:31.853536302 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 15:03:33.391595756 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 15:03:33.393028686 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 15:03:33.394445923 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 15:03:33.396195529 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 15:03:33.399561398 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 15:03:33.405427623 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 15:03:34.068342075 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 15:03:35.733000 54788 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:03:35.733000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54878 closing signal SIGTERM -W0826 15:03:35.734000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54879 closing signal SIGTERM -W0826 15:03:35.734000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54880 closing signal SIGTERM -W0826 15:03:35.735000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54881 closing signal SIGTERM -W0826 15:03:35.735000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54882 closing signal SIGTERM -W0826 15:03:35.736000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54883 closing signal SIGTERM -W0826 15:03:35.736000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54884 closing signal SIGTERM -W0826 15:03:35.736000 54788 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 54885 closing signal SIGTERM +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=23 total_expected_recv=1131026 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269056 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 15:49:37] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 638.9 | throughput per GPU (TFLOP/s/GPU): 199.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.624613E+00 | loss scale: 1.0 | grad norm: 71.371 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +[after training is done] datetime: 2026-08-26 15:49:37 +[rank0]:[W826 15:49:38.491090797 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:49:39.909212188 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:49:39.910546881 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:49:39.911900949 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:49:39.913754563 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:49:40.110953883 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:49:40.440681006 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:49:40.622106427 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:49:42.767000 140201 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:49:42.767000 140201 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 140291 closing signal SIGTERM +W0826 15:49:42.768000 140201 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 140292 closing signal SIGTERM +W0826 15:49:42.768000 140201 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 140293 closing signal SIGTERM +W0826 15:49:42.769000 140201 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 140294 closing signal SIGTERM +W0826 15:49:42.769000 140201 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 140295 closing signal SIGTERM +W0826 15:49:42.770000 140201 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 140296 closing signal SIGTERM +W0826 15:49:42.770000 140201 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 140297 closing signal SIGTERM +W0826 15:49:42.771000 140201 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 140298 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -12746,4 +12746,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 54788 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 140201 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..005fddd70a4684a9e8fe58da0a9f039ea2a3ed7b --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 27 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 2225 +1, 2225 +2, 2225 +3, 2225 +4, 2225 +5, 2225 +6, 2225 +7, 2225 +0, 9077 +1, 9157 +2, 9117 +3, 8993 +4, 9077 +5, 8691 +6, 8945 +7, 9005 +0, 28837 +1, 28937 +2, 28815 +3, 28755 +4, 28767 +5, 28749 +6, 28917 +7, 28939 +0, 38971 +1, 39063 +2, 38909 +3, 38901 +4, 38903 +5, 38889 +6, 39009 +7, 39035 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39781 +1, 39865 +2, 39715 +3, 39715 +4, 39715 +5, 39711 +6, 39847 +7, 39865 +0, 39781 +1, 39865 +2, 39715 +3, 39715 +4, 39715 +5, 39711 +6, 39847 +7, 39865 +0, 111653 +1, 111775 +2, 111677 +3, 111635 +4, 111681 +5, 111613 +6, 111739 +7, 111773 +0, 112149 +1, 112235 +2, 112149 +3, 112083 +4, 112147 +5, 112081 +6, 112215 +7, 112235 +0, 112155 +1, 112239 +2, 112153 +3, 112089 +4, 112153 +5, 112085 +6, 112221 +7, 112239 +0, 114413 +1, 112461 +2, 114397 +3, 114387 +4, 114489 +5, 114523 +6, 114519 +7, 112465 +0, 115241 +1, 113277 +2, 115239 +3, 115175 +4, 115239 +5, 115171 +6, 115307 +7, 113277 +0, 115997 +1, 113979 +2, 115887 +3, 115821 +4, 115955 +5, 115851 +6, 115995 +7, 113961 +0, 127845 +1, 127923 +2, 129893 +3, 129825 +4, 129889 +5, 129817 +6, 129959 +7, 129971 +0, 127845 +1, 127923 +2, 129893 +3, 129825 +4, 129889 +5, 129817 +6, 129959 +7, 129971 +0, 127845 +1, 127923 +2, 129893 +3, 129825 +4, 129889 +5, 129817 +6, 129959 +7, 129971 +0, 127331 +1, 127249 +2, 129593 +3, 129431 +4, 129589 +5, 129397 +6, 129117 +7, 129671 +0, 124071 +1, 124213 +2, 126377 +3, 126309 +4, 126373 +5, 126427 +6, 126613 +7, 126517 +0, 122649 +1, 122465 +2, 124697 +3, 124367 +4, 124693 +5, 124295 +6, 124181 +7, 124775 +0, 1 +1, 3 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 124127 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..5991aa9eab2ba498263394fa74093b9a9a9593d9 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=768 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..bfe62a321be4d9926b4f30887f63db9e90750ca4 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,12749 @@ +[15:49:47] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + DSV2_LITE_L24 / ultraep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:49:48.649000 143370 torch/distributed/run.py:766] +W0826 15:49:48.649000 143370 torch/distributed/run.py:766] ***************************************** +W0826 15:49:48.649000 143370 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:49:48.649000 143370 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 2 is connected to 37 is connected to peer ranks. 7[Gloo] Rank [Gloo] Rank Expected number of connected peer ranks is : [Gloo] Rank peer ranks. 567Expected number of connected peer ranks is : 4 is connected to is connected to +7 is connected to +777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 + + +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 7 +0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 42 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 15:50:06.936450409 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:50:06.936485280 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:50:06.936514738 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:50:06.936567081 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:50:06.936604595 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:50:06.936622342 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:50:06.936723867 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:50:07.097584506 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.562 seconds +time to initialize megatron (seconds): 7.075 +[after megatron is initialized] datetime: 2026-08-26 15:50:12 +building GPT model ... +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4188125184 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:50:16 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:50:16 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3840.71, 3851.77) + train/valid/test-data-iterators-setup ..........: (70.49, 87.08) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 15:50:16 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49060 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49163 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48835 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49061 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49404 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49163 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49061 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49404 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49250 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49250 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48991 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49619 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48913 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48913 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49619 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48761 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48761 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49159 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48753 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49159 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48753 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49552 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49254 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49552 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49254 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49245 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49245 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49315 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48767 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49315 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48767 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49397 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49397 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49059 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49059 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49485 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48801 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49485 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48801 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49157 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49157 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48944 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48944 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48800 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48981 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48800 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48981 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48811 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48811 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49305 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49215 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49305 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48625 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48625 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49351 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49351 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49112 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49112 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49482 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48707 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49359 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49482 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48707 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49030 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49359 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49030 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48995 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48995 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49413 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49413 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49283 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49075 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49075 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49425 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49425 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49206 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49206 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48992 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48992 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49103 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49103 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49396 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49501 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49396 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49501 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49025 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49025 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48764 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49605 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48764 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48854 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49605 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48854 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49479 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49479 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49517 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49390 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49517 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48505 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49390 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48505 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49317 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49317 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48997 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49068 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48982 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48997 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48812 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49068 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48982 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48812 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48971 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48876 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48876 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48790 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48790 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48819 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48819 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49426 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49414 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49426 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49414 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49585 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49585 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49058 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49058 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49430 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49430 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49309 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49293 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49309 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49293 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49477 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49477 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49060 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=23 total_expected_recv=1130161 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49482 required_capacity=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=23 total_expected_recv=1128370 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272512 peak_demand=49413 required_capacity=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=23 total_expected_recv=1130857 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49416 required_capacity=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=23 total_expected_recv=1131337 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=23 total_expected_recv=1130990 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269184 peak_demand=49605 required_capacity=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=23 total_expected_recv=1130414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49398 required_capacity=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=23 total_expected_recv=1129903 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49552 required_capacity=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=23 total_expected_recv=1131936 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49644 required_capacity=1.010010 +Number of parameters in transformer block in billions: 13.53 [2026-08-26 15:50:44] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 27667.7 | throughput per GPU (TFLOP/s/GPU): 4.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092169E+01 | loss scale: 1.0 | grad norm: 30.280 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 37109.60302734375 | max allocated: 104508.90771484375 | reserved: 107726.0 | max reserved: 107726.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49302 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49150 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49095 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49302 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=48980 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49150 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49095 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=48980 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48791 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48791 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49289 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49289 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49309 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49395 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48839 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49309 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48996 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48839 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48996 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49081 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49504 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49081 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49504 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48931 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49452 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48931 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49452 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48684 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49361 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48684 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49170 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49342 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49170 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49342 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49376 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49376 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48853 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48815 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48815 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48982 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48982 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49527 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49198 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49527 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49198 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49435 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49435 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48679 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48679 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49550 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49550 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49245 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48749 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49245 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48749 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49135 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49568 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49162 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49568 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49162 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48813 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48813 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49030 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49030 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49014 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49014 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49411 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49411 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49119 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48958 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48958 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49447 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49447 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49095 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49095 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49425 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49425 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48908 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48882 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49308 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48882 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49308 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49293 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49293 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49349 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49349 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48839 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48839 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49158 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49158 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48973 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48973 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48834 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49477 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48834 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49477 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49584 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49584 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49098 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49098 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48869 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49025 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48869 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48948 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49025 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48948 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=48717 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49508 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=48717 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49508 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49339 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49396 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49339 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49273 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49396 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48817 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49273 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48817 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48824 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48974 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48824 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49412 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48974 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49412 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=48703 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=48703 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49202 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49293 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49202 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49293 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49255 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49255 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49193 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49193 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48729 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48729 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49352 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49352 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=48686 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=48686 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49235 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49235 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=23 total_expected_recv=1130967 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49584 required_capacity=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=23 total_expected_recv=1130188 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49416 required_capacity=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=23 total_expected_recv=1130295 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49452 required_capacity=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=23 total_expected_recv=1131840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269056 peak_demand=49568 required_capacity=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=23 total_expected_recv=1131667 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49458 required_capacity=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=23 total_expected_recv=1129272 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272000 peak_demand=49456 required_capacity=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=23 total_expected_recv=1127857 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273792 peak_demand=49493 required_capacity=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=23 total_expected_recv=1131882 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49508 required_capacity=1.007243 + [2026-08-26 15:50:44] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 635.1 | throughput per GPU (TFLOP/s/GPU): 200.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091868E+01 | loss scale: 1.0 | grad norm: 30.184 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49122 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49532 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49464 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48705 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48660 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49122 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49532 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49464 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48705 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48660 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49344 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48863 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49344 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49189 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48863 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49189 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49417 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49347 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49417 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49419 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49163 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49419 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49163 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49385 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48704 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49385 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48704 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49535 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48586 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49535 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48586 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49554 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48771 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49554 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48771 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49601 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49601 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49463 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48972 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49463 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48972 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49213 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49213 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48753 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48753 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49577 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49577 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49201 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49201 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48837 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49170 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48837 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49170 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49527 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49527 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49385 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49385 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49102 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49102 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49236 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49236 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48941 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49504 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48941 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49504 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48792 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48792 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49189 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49442 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49189 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49442 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49131 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49420 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49420 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=61440 expected_recv=48797 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=61440 expected_recv=48797 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48903 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48903 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49526 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49084 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48921 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48921 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49600 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48939 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49058 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49600 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49058 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49054 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49054 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=61440 expected_recv=48996 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=61440 expected_recv=48996 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49058 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49058 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49371 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49371 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48801 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48801 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49394 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49394 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49431 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49431 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49027 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49155 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48754 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49235 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49027 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49155 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48754 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49235 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49186 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49186 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49400 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48884 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48884 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49400 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49472 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49472 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48870 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49308 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48870 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49308 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48959 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48959 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48926 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48926 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49354 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49380 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49354 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49380 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48797 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48797 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=23 total_expected_recv=1131007 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49535 required_capacity=1.007792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=23 total_expected_recv=1131856 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268928 peak_demand=49601 required_capacity=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=23 total_expected_recv=1130704 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49526 required_capacity=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=23 total_expected_recv=1130378 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271360 peak_demand=49577 required_capacity=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=23 total_expected_recv=1128434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272768 peak_demand=49464 required_capacity=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=23 total_expected_recv=1129402 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272768 peak_demand=49420 required_capacity=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=23 total_expected_recv=1130448 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=23 total_expected_recv=1131739 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269440 peak_demand=49554 required_capacity=1.008179 + [2026-08-26 15:50:45] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 625.9 | throughput per GPU (TFLOP/s/GPU): 203.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.944945E+00 | loss scale: 1.0 | grad norm: 41.756 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49304 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49016 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49273 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49304 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49016 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49273 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49339 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49339 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48834 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48834 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=48974 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=48974 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48919 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49514 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48919 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49514 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49215 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49084 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49552 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49552 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48816 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48816 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49166 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=48716 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49166 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=48716 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49379 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49355 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49355 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49365 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49235 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49365 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49344 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49235 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49344 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48994 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48994 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49004 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49512 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49004 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49512 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49340 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48677 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49340 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48677 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49542 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49542 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48843 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48843 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49453 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48712 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49453 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48712 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48978 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48978 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49561 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49561 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48540 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48540 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49076 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49494 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49076 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49494 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=48903 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=48903 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49206 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49363 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49206 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49363 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49429 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49313 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49429 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49313 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48914 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49059 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48914 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49059 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49225 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49225 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49403 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49403 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49337 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49337 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49103 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49103 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49261 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49261 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48933 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48933 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48984 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49298 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49298 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48934 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49363 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48934 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49363 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48763 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49326 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48763 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48893 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49567 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48893 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49207 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49567 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49207 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49426 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49426 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48856 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49404 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49302 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48856 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49404 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49119 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49302 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48939 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49315 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49315 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48956 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49394 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49394 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49038 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49038 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48964 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49326 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48964 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=23 total_expected_recv=1128777 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273024 peak_demand=49542 required_capacity=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=23 total_expected_recv=1130414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49404 required_capacity=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=23 total_expected_recv=1130532 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=23 total_expected_recv=1130977 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49561 required_capacity=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=23 total_expected_recv=1131080 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=23 total_expected_recv=1129971 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49460 required_capacity=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=23 total_expected_recv=1131011 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270208 peak_demand=49567 required_capacity=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=23 total_expected_recv=1131206 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270720 peak_demand=49439 required_capacity=1.005839 + [2026-08-26 15:50:46] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 624.6 | throughput per GPU (TFLOP/s/GPU): 204.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.474912E+00 | loss scale: 1.0 | grad norm: 7.074 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49234 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49442 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48887 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49234 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49442 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48887 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48879 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49466 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48879 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48885 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49466 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48886 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49507 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48886 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49419 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49054 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49507 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49419 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49054 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49004 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49004 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48933 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49253 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48933 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48835 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49501 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49501 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49516 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49197 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49516 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49437 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49437 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48912 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49471 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49253 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48912 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49471 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49252 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49252 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48879 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49405 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48879 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49405 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48820 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48820 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49379 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48808 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48808 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49601 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48953 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49006 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49601 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49423 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48953 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49006 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49203 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49423 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49203 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49497 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49497 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48831 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48831 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48921 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48921 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49532 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49429 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48965 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49345 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49532 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49429 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48965 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49345 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48871 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48871 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49291 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49566 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48851 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49291 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49566 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49135 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49320 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48592 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49320 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48592 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49253 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=48778 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49359 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=48778 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49359 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49345 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49470 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49047 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49345 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49470 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49047 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48849 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48849 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49119 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49480 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49480 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49347 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49157 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49157 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=48944 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=48944 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49411 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49411 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48863 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48863 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49369 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49369 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49466 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49466 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49255 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49255 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48851 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=23 total_expected_recv=1131626 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269952 peak_demand=49507 required_capacity=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=23 total_expected_recv=1129438 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49566 required_capacity=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=23 total_expected_recv=1129671 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49480 required_capacity=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=23 total_expected_recv=1131106 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=23 total_expected_recv=1129383 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49429 required_capacity=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=23 total_expected_recv=1129952 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270720 peak_demand=49497 required_capacity=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=23 total_expected_recv=1131458 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49601 required_capacity=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=23 total_expected_recv=1131334 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269952 peak_demand=49516 required_capacity=1.007406 + [2026-08-26 15:50:46] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 629.6 | throughput per GPU (TFLOP/s/GPU): 202.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119712E+01 | loss scale: 1.0 | grad norm: 398.552 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48880 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49276 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48880 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49276 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49438 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49438 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48909 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48909 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49505 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49008 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49505 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49008 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48810 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49274 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48788 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48810 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49274 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48788 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49060 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49068 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49337 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=48773 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49340 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49068 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49337 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=48773 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49340 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48748 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49358 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48748 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49358 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49201 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49626 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49201 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49626 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49236 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48765 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49236 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48765 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48871 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49565 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48871 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48893 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49565 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48893 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49198 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49198 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49131 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49328 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49328 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49436 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49436 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48945 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49418 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48843 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49418 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48843 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49430 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49430 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49150 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49150 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49067 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49067 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=48882 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49286 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=48882 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49286 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49561 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49561 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=48719 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=48719 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49474 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49474 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49503 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49386 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=48632 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49386 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49503 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=48632 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49577 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49577 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49045 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48948 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49045 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48948 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48800 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48800 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48984 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48788 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48788 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49517 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49517 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49358 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49358 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49365 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49254 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49365 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49254 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48956 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49193 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49193 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48905 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48972 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48905 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48972 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48857 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49546 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48857 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49492 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49492 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=48734 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=48734 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=23 total_expected_recv=1131183 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49577 required_capacity=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=23 total_expected_recv=1132379 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268544 peak_demand=49565 required_capacity=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=23 total_expected_recv=1132087 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=23 total_expected_recv=1129768 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49561 required_capacity=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=23 total_expected_recv=1129031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272512 peak_demand=49436 required_capacity=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=23 total_expected_recv=1127234 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274688 peak_demand=49296 required_capacity=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=23 total_expected_recv=1131636 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=23 total_expected_recv=1130650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49644 required_capacity=1.010010 + [2026-08-26 15:50:47] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 625.0 | throughput per GPU (TFLOP/s/GPU): 204.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.384323E+00 | loss scale: 1.0 | grad norm: 5.040 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49408 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49573 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49008 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49408 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49573 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49008 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49307 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49340 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49307 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49340 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49038 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49038 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49393 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49393 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48858 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48858 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49550 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48906 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49550 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48906 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49026 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49274 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49026 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49274 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49379 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49008 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49355 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49008 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48646 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49355 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48646 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48797 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49354 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48797 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49354 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49030 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49030 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49118 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48857 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49291 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48857 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49352 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49291 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49248 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49537 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49352 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49248 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49537 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=48717 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=48717 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49453 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49297 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49453 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49297 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48937 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48885 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49207 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49207 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49467 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49467 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49509 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48801 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49509 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48801 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48892 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49127 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48892 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49127 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48858 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48858 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49414 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49414 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49118 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49318 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49318 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48810 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48810 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49254 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49254 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49393 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49350 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49084 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49393 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49350 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49298 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49298 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49217 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49217 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48854 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49432 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49358 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48854 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49358 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48804 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49432 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48804 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49309 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49402 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49309 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49402 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49064 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49338 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49064 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49338 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49013 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49013 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49438 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49430 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48605 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49438 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49430 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48605 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48809 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49419 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49528 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49414 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48809 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49419 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49528 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49414 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49202 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48848 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49202 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48848 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49217 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49217 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48736 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48736 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49021 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49441 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49318 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49580 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49021 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49441 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49318 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49580 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=23 total_expected_recv=1131843 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=23 total_expected_recv=1129374 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49573 required_capacity=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=23 total_expected_recv=1129898 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49537 required_capacity=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=23 total_expected_recv=1131168 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=23 total_expected_recv=1131681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269184 peak_demand=49580 required_capacity=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=23 total_expected_recv=1129454 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272000 peak_demand=49550 required_capacity=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=23 total_expected_recv=1130061 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49467 required_capacity=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=23 total_expected_recv=1130489 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49644 required_capacity=1.010010 + [2026-08-26 15:50:48] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 624.6 | throughput per GPU (TFLOP/s/GPU): 204.2 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.983669E+00 | loss scale: 1.0 | grad norm: 3.614 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48848 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48948 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49350 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49273 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48848 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49373 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48948 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49273 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49350 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49373 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49318 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49318 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49002 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49002 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49162 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48787 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49162 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48787 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49342 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49342 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48689 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49016 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49252 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49433 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48689 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49016 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49252 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49433 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48885 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=48709 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=48709 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49331 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49331 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49417 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49417 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48837 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49285 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49041 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48837 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49285 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49041 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49225 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49225 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48950 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48950 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48771 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48771 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49361 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49295 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49295 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49307 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49307 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49474 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49035 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49233 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49474 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49035 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=48928 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=48928 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49505 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49505 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48975 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48975 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48939 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49197 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48792 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48792 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49011 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49011 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49446 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49446 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48969 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48969 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49413 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49413 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49183 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49466 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49183 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49466 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48932 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48932 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48848 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48848 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=48802 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=48802 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49442 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49442 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49135 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49158 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49158 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48981 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48981 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48864 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48908 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48864 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49406 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49400 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49406 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49400 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48810 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48810 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49436 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49436 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49407 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49407 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49417 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49295 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49417 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48971 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49295 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48773 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48877 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48773 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48877 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49473 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48876 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49473 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48876 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49537 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48689 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49537 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48689 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49183 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49183 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49279 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49279 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49378 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49378 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49512 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49312 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49512 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49312 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49262 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48798 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49262 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48798 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49045 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48440 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49045 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49314 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48440 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49490 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49314 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49490 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49315 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49315 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48991 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49313 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49313 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=23 total_expected_recv=1128840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272000 peak_demand=49505 required_capacity=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=23 total_expected_recv=1131448 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269440 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=23 total_expected_recv=1130060 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=23 total_expected_recv=1131595 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49474 required_capacity=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=23 total_expected_recv=1129947 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49512 required_capacity=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=23 total_expected_recv=1129953 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49446 required_capacity=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=23 total_expected_recv=1131368 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270080 peak_demand=49436 required_capacity=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=23 total_expected_recv=1130757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49490 required_capacity=1.006877 + [2026-08-26 15:50:48] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 624.7 | throughput per GPU (TFLOP/s/GPU): 204.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.792192E+00 | loss scale: 1.0 | grad norm: 5.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48867 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48867 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48945 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49333 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49333 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48831 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48831 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49006 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49006 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49233 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48825 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48825 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49540 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49540 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48707 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48707 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48791 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48791 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49407 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49407 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49025 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49025 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48906 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48906 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49203 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49461 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49203 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49461 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49553 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49023 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48978 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49553 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49515 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49023 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48978 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49515 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48820 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48820 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49444 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49444 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49210 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49333 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49210 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49333 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48933 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48933 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49186 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49035 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49186 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49479 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49035 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49479 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=48923 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=48923 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49406 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49119 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49406 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49262 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48942 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49307 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48942 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49262 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49307 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48775 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49491 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48775 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49491 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48791 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48791 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49061 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49061 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49546 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49381 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=48872 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48583 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=48872 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48583 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49450 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49450 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48842 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48842 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48659 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49463 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48659 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48992 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49463 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48992 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=48831 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49289 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=48831 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49289 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49112 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49473 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49112 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49473 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=23 total_expected_recv=1131511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269824 peak_demand=49491 required_capacity=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=23 total_expected_recv=1129237 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272512 peak_demand=49479 required_capacity=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=23 total_expected_recv=1129458 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49463 required_capacity=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=23 total_expected_recv=1131442 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270080 peak_demand=49399 required_capacity=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=23 total_expected_recv=1130987 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=23 total_expected_recv=1131882 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268800 peak_demand=49540 required_capacity=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=23 total_expected_recv=1128828 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273280 peak_demand=49553 required_capacity=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=23 total_expected_recv=1130623 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49546 required_capacity=1.008016 + [2026-08-26 15:50:49] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 624.9 | throughput per GPU (TFLOP/s/GPU): 204.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.617704E+00 | loss scale: 1.0 | grad norm: 312.445 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49324 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48860 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49324 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48850 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48860 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48850 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49081 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49081 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49163 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49163 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49045 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49045 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49391 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49391 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48945 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48818 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48818 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48768 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49418 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48768 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49554 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49418 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49554 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48835 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49459 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49459 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48691 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48691 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49276 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49381 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49510 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49276 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49510 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49020 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49210 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49020 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48719 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49210 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48719 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49312 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49312 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48847 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49393 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48847 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49393 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49389 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49314 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49389 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49314 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49507 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48975 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49507 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48975 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48764 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48764 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48698 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49156 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48698 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49289 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49374 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49289 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49374 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48803 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49342 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48803 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49342 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48980 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49413 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49034 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48980 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49413 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49034 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49295 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49162 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49295 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49162 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49352 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49245 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49352 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49245 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48861 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48861 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49408 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48954 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49408 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48969 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48954 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48969 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48941 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49298 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48941 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49298 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48845 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49156 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48845 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49011 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49011 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49569 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49569 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=48889 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48796 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=48889 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48796 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49234 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49234 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48942 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48942 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49488 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49488 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48913 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48913 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48703 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48703 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49549 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49005 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49549 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49005 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49403 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49403 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49462 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49462 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49433 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49075 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49433 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49000 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49075 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49000 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49462 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48776 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49462 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48776 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48958 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49490 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48958 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49490 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49041 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48932 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49041 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48928 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48932 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48928 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49410 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49410 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=23 total_expected_recv=1130315 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49510 required_capacity=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=23 total_expected_recv=1128697 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273536 peak_demand=49389 required_capacity=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=23 total_expected_recv=1131501 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269824 peak_demand=49554 required_capacity=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=23 total_expected_recv=1130088 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49413 required_capacity=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=23 total_expected_recv=1131104 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49456 required_capacity=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=23 total_expected_recv=1129686 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272640 peak_demand=49418 required_capacity=1.005412 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=23 total_expected_recv=1131660 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269440 peak_demand=49569 required_capacity=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=23 total_expected_recv=1130917 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269824 peak_demand=49549 required_capacity=1.008077 + [2026-08-26 15:50:49] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 625.6 | throughput per GPU (TFLOP/s/GPU): 203.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.605448E+00 | loss scale: 1.0 | grad norm: 105.043 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49447 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49371 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49371 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49447 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49159 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49038 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49297 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49159 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49038 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49297 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=48624 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49351 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=48624 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49351 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49156 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48973 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48973 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49250 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48926 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49250 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48926 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48806 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48806 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49574 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49362 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49574 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49213 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49362 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49213 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49197 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49026 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49165 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49026 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49482 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49165 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49274 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49482 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49013 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49274 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49013 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48964 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48964 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49555 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49555 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49372 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49372 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48951 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48951 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48830 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48830 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48815 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48815 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49354 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49583 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49354 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49583 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48806 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48806 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49285 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49287 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49285 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49287 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49283 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49193 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49193 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48826 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48826 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48814 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48814 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49526 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=48991 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49347 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48982 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49180 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48982 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49180 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48954 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49548 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48954 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49548 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=48838 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48861 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=48838 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48861 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49385 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49385 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49095 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49095 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48862 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48862 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49538 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49437 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49037 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49538 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49041 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49437 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49037 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49041 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49304 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49304 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=48700 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=48700 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49395 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49027 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49027 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49349 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49349 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48913 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48913 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48978 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48978 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49518 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49518 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49565 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49565 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49131 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49034 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49034 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49464 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48956 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49464 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49064 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49064 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49412 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49412 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49486 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49486 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49157 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49157 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=23 total_expected_recv=1128996 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49565 required_capacity=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=23 total_expected_recv=1130067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270720 peak_demand=49412 required_capacity=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=23 total_expected_recv=1132222 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268928 peak_demand=49583 required_capacity=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=23 total_expected_recv=1130787 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49574 required_capacity=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=23 total_expected_recv=1130963 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49538 required_capacity=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=23 total_expected_recv=1129336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49385 required_capacity=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=23 total_expected_recv=1130595 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271360 peak_demand=49486 required_capacity=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=23 total_expected_recv=1131002 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49644 required_capacity=1.010010 + [2026-08-26 15:50:50] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 624.9 | throughput per GPU (TFLOP/s/GPU): 204.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.583566E+00 | loss scale: 1.0 | grad norm: 3.092 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49059 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48753 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49364 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49362 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49059 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48753 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49364 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49362 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49410 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48849 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49410 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48849 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49287 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49511 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49287 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49511 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49023 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49023 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48716 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48716 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49047 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49035 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49047 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49035 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49503 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49529 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49503 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48609 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49529 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48609 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49586 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49586 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49203 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48926 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49203 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48926 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48870 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48870 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49308 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48894 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49308 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48894 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49279 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49279 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48847 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48984 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48790 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49572 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48847 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48790 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49572 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49459 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49459 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48937 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49283 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48857 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49155 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49286 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48857 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49155 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49404 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49286 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49404 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48971 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49572 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49572 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49068 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49014 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=48794 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49068 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49014 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=48794 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49478 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49478 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=61440 expected_recv=48963 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49363 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=61440 expected_recv=48963 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49363 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49004 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49004 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49438 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48937 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48802 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49438 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48802 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48781 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48781 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49253 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49574 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49494 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49574 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49494 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49593 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48814 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49593 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48814 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48770 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49333 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48770 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49279 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49333 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49279 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49454 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49165 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49454 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49165 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49361 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48820 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48820 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48808 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48808 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48642 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49392 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48642 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49392 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49400 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49400 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49086 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49086 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49170 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49170 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49217 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49076 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49217 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49076 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48659 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48906 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48659 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48906 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49020 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49020 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49034 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49034 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48816 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48816 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=23 total_expected_recv=1130115 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49529 required_capacity=1.007670 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=23 total_expected_recv=1131219 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270208 peak_demand=49459 required_capacity=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=23 total_expected_recv=1130661 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49593 required_capacity=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=23 total_expected_recv=1129200 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49494 required_capacity=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=23 total_expected_recv=1130191 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49574 required_capacity=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=23 total_expected_recv=1131777 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49586 required_capacity=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=23 total_expected_recv=1130679 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49511 required_capacity=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=23 total_expected_recv=1130126 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49644 required_capacity=1.010010 + [2026-08-26 15:50:51] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 631.1 | throughput per GPU (TFLOP/s/GPU): 202.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.183624E+00 | loss scale: 1.0 | grad norm: 1341.192 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +[after training is done] datetime: 2026-08-26 15:50:51 +[rank0]:[W826 15:50:52.412386877 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:50:53.876767375 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:50:53.890833892 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:50:53.892518724 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:50:53.894329150 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:50:53.894453489 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:50:53.900970903 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:50:53.984365093 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:50:57.342000 143370 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:50:57.343000 143370 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 143483 closing signal SIGTERM +W0826 15:50:57.343000 143370 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 143484 closing signal SIGTERM +W0826 15:50:57.344000 143370 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 143485 closing signal SIGTERM +W0826 15:50:57.344000 143370 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 143486 closing signal SIGTERM +W0826 15:50:57.345000 143370 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 143487 closing signal SIGTERM +W0826 15:50:57.345000 143370 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 143488 closing signal SIGTERM +W0826 15:50:57.345000 143370 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 143489 closing signal SIGTERM +W0826 15:50:57.346000 143370 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 143490 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 143370 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv index 20f4b0a3d693a2f5a3851606effe2433ad211900..b16bee982c5bbce35135b5ee0f4d20ef30209b24 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv @@ -26,7 +26,7 @@ 1, 4 2, 4 3, 4 -4, 2 +4, 4 5, 1 6, 1 7, 1 @@ -39,10 +39,10 @@ 6, 4 7, 4 0, 4 -1, 4 -2, 4 +1, 47 +2, 629 3, 4 -4, 629 +4, 4 5, 4 6, 4 7, 4 @@ -62,14 +62,14 @@ 5, 1803 6, 1803 7, 1803 -0, 2393 -1, 2387 -2, 2371 -3, 2385 -4, 2385 -5, 2397 -6, 2393 -7, 2377 +0, 2443 +1, 2443 +2, 2443 +3, 2443 +4, 2441 +5, 2443 +6, 2443 +7, 2443 0, 46703 1, 46137 2, 46137 @@ -78,75 +78,115 @@ 5, 46837 6, 46837 7, 46837 -0, 47107 -1, 46537 -2, 46537 -3, 47237 -4, 47237 -5, 47237 -6, 47237 -7, 47237 -0, 47497 -1, 46927 -2, 46927 -3, 47627 -4, 47627 -5, 47627 -6, 47627 -7, 47627 -0, 47497 -1, 46927 -2, 46927 -3, 47627 -4, 47627 -5, 47627 -6, 47627 -7, 47627 -0, 47497 -1, 46927 -2, 46927 -3, 47627 -4, 47627 -5, 47627 -6, 47627 -7, 47627 -0, 47497 -1, 46927 -2, 46927 -3, 47627 -4, 47627 -5, 47627 -6, 47627 -7, 47627 -0, 47497 -1, 46927 -2, 46927 -3, 47627 -4, 47627 -5, 47627 -6, 47627 -7, 47627 -0, 46851 -1, 46345 -2, 46927 -3, 47301 -4, 47045 -5, 47627 -6, 47301 -7, 46981 -0, 2561 -1, 2561 -2, 2561 -3, 2625 -4, 46981 -5, 2884 -6, 2564 -7, 2564 -0, 0 -1, 0 -2, 0 -3, 0 -4, 0 -5, 0 -6, 0 -7, 0 +0, 47109 +1, 46539 +2, 46539 +3, 47243 +4, 47239 +5, 47241 +6, 47239 +7, 47239 +0, 48395 +1, 47825 +2, 47825 +3, 48525 +4, 48525 +5, 48525 +6, 48525 +7, 48525 +0, 48675 +1, 48105 +2, 48105 +3, 48805 +4, 48805 +5, 48805 +6, 48805 +7, 48805 +0, 48675 +1, 48105 +2, 48105 +3, 48805 +4, 48805 +5, 48805 +6, 48805 +7, 48805 +0, 50879 +1, 50309 +2, 50309 +3, 51009 +4, 51009 +5, 51009 +6, 51009 +7, 51009 +0, 58343 +1, 58357 +2, 58373 +3, 58517 +4, 58403 +5, 58311 +6, 58441 +7, 58349 +0, 74975 +1, 75009 +2, 74989 +3, 75019 +4, 75019 +5, 75019 +6, 75019 +7, 75019 +0, 74975 +1, 75009 +2, 74989 +3, 75019 +4, 75019 +5, 75019 +6, 75019 +7, 75019 +0, 74975 +1, 75011 +2, 74991 +3, 75021 +4, 75021 +5, 75021 +6, 75021 +7, 75021 +0, 74975 +1, 75011 +2, 74991 +3, 75021 +4, 75021 +5, 75021 +6, 75021 +7, 75021 +0, 75523 +1, 75597 +2, 75527 +3, 75617 +4, 75571 +5, 75555 +6, 75575 +7, 75577 +0, 82201 +1, 82167 +2, 82031 +3, 82077 +4, 82119 +5, 82099 +6, 82083 +7, 82083 +0, 97625 +1, 97675 +2, 97675 +3, 97551 +4, 97551 +5, 97551 +6, 97551 +7, 97551 +0, 4414 +1, 2814 +2, 2814 +3, 2814 +4, 2814 +5, 2814 +6, 3134 +7, 97551 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env index 6752517ec4555ac82a4ae485c93c65fb9dcba003..c5d9e4fbf6cdf85beff34e20965ab3579d96a120 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env @@ -28,7 +28,7 @@ router_balancing=aux_loss aux_loss_coeff=1e-2 router_pre_softmax=1 router_fusion=0 -router_dtype= +router_dtype=fp32 device_initiated_grouped_gemm=1 norm_epsilon=1e-6 seq_length=4096 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log index ece45ef7ca9a2e8c4e3894574ed214e42055905d..6827215b43c01f7a912bfc4a281fa3f883eb4a17 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log @@ -1,4 +1,4 @@ -[15:04:30] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:51:01] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) ============================================================== QWEN2_57B / ultraep (slots/rank=8 policy=matched) E=64 K=8 H'=2560 m=8 @@ -7,28 +7,23 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-UltraEP @ eaf829e7 ============================================================== -W0826 15:04:31.818000 60221 torch/distributed/run.py:766] -W0826 15:04:31.818000 60221 torch/distributed/run.py:766] ***************************************** -W0826 15:04:31.818000 60221 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:04:31.818000 60221 torch/distributed/run.py:766] ***************************************** +W0826 15:51:03.095000 146553 torch/distributed/run.py:766] +W0826 15:51:03.095000 146553 torch/distributed/run.py:766] ***************************************** +W0826 15:51:03.095000 146553 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:51:03.095000 146553 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None accumulate and all-reduce gradients in fp32 for bfloat16 data type. using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") ------------------------ arguments ------------------------ @@ -343,7 +338,7 @@ using torch.bfloat16 for parameters ... moe_permute_fusion .............................. False moe_received_token_capacity ..................... 1.25 moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None + moe_router_dtype ................................ fp32 moe_router_enable_expert_bias ................... False moe_router_force_load_balancing ................. False moe_router_group_topk ........................... None @@ -578,45 +573,50 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... -[Gloo] Rank [Gloo] Rank [Gloo] Rank 013 is connected to is connected to is connected to 77[Gloo] Rank 7 peer ranks. peer ranks. 2 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : is connected to Expected number of connected peer ranks is : 7777 - peer ranks. - -Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 75 - is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. -Expected number of connected peer ranks is : 7 +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 1 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. [Gloo] Rank +Expected number of connected peer ranks is : 57 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 32 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - +[Gloo] Rank [Gloo] Rank 6 is connected to 72 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank7]:[W826 15:04:49.719767567 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 15:04:49.721747479 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:04:49.721811630 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 15:04:49.721828016 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:04:49.723746853 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:04:49.726046387 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 15:04:49.727355538 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 15:51:21.400908954 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:51:21.401037167 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:51:21.401085339 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:51:21.402604657 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:51:21.402911706 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:51:21.423907607 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:51:21.423949866 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' make: Nothing to be done for 'default'. @@ -624,27 +624,11 @@ make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' >>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:04:49.867202970 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.798 seconds -time to initialize megatron (seconds): 7.096 -[after megatron is initialized] datetime: 2026-08-26 15:04:55 +[rank0]:[W826 15:51:21.585972623 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.841 seconds +time to initialize megatron (seconds): 7.186 +[after megatron is initialized] datetime: 2026-08-26 15:51:27 building GPT model ... -/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( -/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1433: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. - warnings.warn( /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. @@ -692,11 +676,11 @@ Params for bucket 2 (73403904 elements, 73403904 padded size): Params for bucket 3 (146800640 elements, 146800640 padded size): module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.self_attention.linear_qkv.bias module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.bias module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight Params for bucket 5 (146800640 elements, 146800640 padded size): @@ -724,10 +708,10 @@ Params for bucket 9 (146800640 elements, 146800640 padded size): Params for bucket 10 (103001600 elements, 103001600 padded size): module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.linear_qkv.bias - module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight Params for bucket 11 (146800640 elements, 146800640 padded size): module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight @@ -744,31 +728,31 @@ Params for bucket 13 (146800640 elements, 146800640 padded size): Params for bucket 14 (103001600 elements, 103001600 padded size): module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.linear_qkv.bias module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight Params for bucket 15 (146800640 elements, 146800640 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight Params for bucket 16 (103001600 elements, 103001600 padded size): module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.bias module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight Params for bucket 17 (146800640 elements, 146800640 padded size): module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.2.self_attention.linear_qkv.weight Params for bucket 19 (146800640 elements, 146800640 padded size): module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight Params for bucket 20 (103001600 elements, 103001600 padded size): @@ -783,28 +767,28 @@ Params for bucket 21 (146800640 elements, 146800640 padded size): module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight Params for bucket 22 (209890816 elements, 209890816 padded size): module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.linear_proj.weight module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 Params for bucket 1 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 Params for bucket 2 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 Params for bucket 3 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 Params for bucket 4 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 @@ -835,14 +819,14 @@ Params for bucket 9 (45875200 elements, 45875200 padded size): Params for bucket 10 (45875200 elements, 45875200 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 Params for bucket 11 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 Params for bucket 12 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 @@ -891,80 +875,80 @@ Params for bucket 21 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight2 Params for bucket 22 (45875200 elements, 45875200 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 Params for bucket 23 (45875200 elements, 45875200 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 Params for bucket 24 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 Params for bucket 25 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 Params for bucket 26 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 Params for bucket 27 (45875200 elements, 45875200 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 Params for bucket 28 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 Params for bucket 29 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 Params for bucket 30 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 Params for bucket 31 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 Params for bucket 32 (45875200 elements, 45875200 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 Params for bucket 33 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 34 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 Params for bucket 35 (45875200 elements, 45875200 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 37 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 Params for bucket 38 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 @@ -974,29 +958,29 @@ Params for bucket 39 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight4 - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 Params for bucket 41 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 42 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 Params for bucket 43 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight3 - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 Params for bucket 44 (18350080 elements, 18350080 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:04:59 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:51:30 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1004,7 +988,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1017,839 +1001,5067 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:04:59 +[after dataloaders are built] datetime: 2026-08-26 15:51:30 done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3452.85, 3471.97) - train/valid/test-data-iterators-setup ..........: (70.81, 86.46) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 15:04:59 -WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] -WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] -WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] -WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] -WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] -WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] -WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] -WARNING:megatron.core.utils:Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward\n return self.ultraep_forward(hidden_states)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward\n probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute\n return self.runtime.reroute(virtual_layer_id, probs, routing_map)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute\n expanded_probs, expanded_routing_map = self._dense_reroute(\n ^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute\n expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward\n expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', 'RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32\n'] -[rank4]: Traceback (most recent call last): -[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in -[rank4]: pretrain( -[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain -[rank4]: iteration, num_floating_point_operations_so_far = train( -[rank4]: ^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train -[rank4]: ) = train_step( -[rank4]: ^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step -[rank4]: losses_reduced = forward_backward_func( -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank4]: output_tensor, num_tokens = forward_step( -[rank4]: ^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank4]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step -[rank4]: output_tensor = model( -[rank4]: ^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank4]: return forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank4]: return self.module(*inputs, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank4]: return forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward -[rank4]: outputs = self.module(*inputs, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank4]: return forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank4]: hidden_states = self.decoder( -[rank4]: ^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank4]: return forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank4]: hidden_states, context = layer( -[rank4]: ^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ -[rank4]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank4]: return forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank4]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp -[rank4]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank4]: return self._call_impl(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank4]: return forward_call(*args, **kwargs) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward -[rank4]: return self.ultraep_forward(hidden_states) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward -[rank4]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute -[rank4]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute -[rank4]: expanded_probs, expanded_routing_map = self._dense_reroute( -[rank4]: ^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute -[rank4]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank4]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward -[rank4]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( -[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank4]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -[rank7]: Traceback (most recent call last): -[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in -[rank7]: pretrain( -[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain -[rank7]: iteration, num_floating_point_operations_so_far = train( -[rank7]: ^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train -[rank7]: ) = train_step( -[rank7]: ^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step -[rank7]: losses_reduced = forward_backward_func( -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank7]: output_tensor, num_tokens = forward_step( -[rank7]: ^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank7]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step -[rank7]: output_tensor = model( -[rank7]: ^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank7]: return self._call_impl(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank7]: return forward_call(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank7]: return self.module(*inputs, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank7]: return self._call_impl(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank7]: return forward_call(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward -[rank7]: outputs = self.module(*inputs, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank7]: return self._call_impl(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank7]: return forward_call(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank7]: hidden_states = self.decoder( -[rank7]: ^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank7]: return self._call_impl(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank7]: return forward_call(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank7]: hidden_states, context = layer( -[rank7]: ^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ -[rank7]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank7]: return self._call_impl(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank7]: return forward_call(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank7]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp -[rank7]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank7]: return self._call_impl(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank7]: return forward_call(*args, **kwargs) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward -[rank7]: return self.ultraep_forward(hidden_states) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward -[rank7]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute -[rank7]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute -[rank7]: expanded_probs, expanded_routing_map = self._dense_reroute( -[rank7]: ^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute -[rank7]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank7]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward -[rank7]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( -[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank7]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -[rank1]: Traceback (most recent call last): -[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in -[rank1]: pretrain( -[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: ^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train -[rank1]: ) = train_step( -[rank1]: ^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: ^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step -[rank1]: output_tensor = model( -[rank1]: ^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank1]: hidden_states = self.decoder( -[rank1]: ^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank1]: hidden_states, context = layer( -[rank1]: ^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp -[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward -[rank1]: return self.ultraep_forward(hidden_states) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward -[rank1]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute -[rank1]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute -[rank1]: expanded_probs, expanded_routing_map = self._dense_reroute( -[rank1]: ^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute -[rank1]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward -[rank1]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -[rank3]: Traceback (most recent call last): -[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in -[rank3]: pretrain( -[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain -[rank3]: iteration, num_floating_point_operations_so_far = train( -[rank3]: ^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train -[rank3]: ) = train_step( -[rank3]: ^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step -[rank3]: losses_reduced = forward_backward_func( -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank3]: output_tensor, num_tokens = forward_step( -[rank3]: ^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step -[rank3]: output_tensor = model( -[rank3]: ^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank3]: return forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank3]: return self.module(*inputs, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank3]: return forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward -[rank3]: outputs = self.module(*inputs, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank3]: return forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank3]: hidden_states = self.decoder( -[rank3]: ^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank3]: return forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank3]: hidden_states, context = layer( -[rank3]: ^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ -[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank3]: return forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp -[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank3]: return forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward -[rank3]: return self.ultraep_forward(hidden_states) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward -[rank3]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute -[rank3]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute -[rank3]: expanded_probs, expanded_routing_map = self._dense_reroute( -[rank3]: ^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute -[rank3]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward -[rank3]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -[rank2]: Traceback (most recent call last): -[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in -[rank2]: pretrain( -[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain -[rank2]: iteration, num_floating_point_operations_so_far = train( -[rank2]: ^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train -[rank2]: ) = train_step( -[rank2]: ^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step -[rank2]: losses_reduced = forward_backward_func( -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank2]: output_tensor, num_tokens = forward_step( -[rank2]: ^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank2]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step -[rank2]: output_tensor = model( -[rank2]: ^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank2]: return forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank2]: return self.module(*inputs, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank2]: return forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward -[rank2]: outputs = self.module(*inputs, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank2]: return forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank2]: hidden_states = self.decoder( -[rank2]: ^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank2]: return forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank2]: hidden_states, context = layer( -[rank2]: ^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ -[rank2]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank2]: return forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank2]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp -[rank2]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank2]: return self._call_impl(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank2]: return forward_call(*args, **kwargs) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward -[rank2]: return self.ultraep_forward(hidden_states) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward -[rank2]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute -[rank2]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute -[rank2]: expanded_probs, expanded_routing_map = self._dense_reroute( -[rank2]: ^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute -[rank2]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank2]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward -[rank2]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( -[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank2]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -[rank5]: Traceback (most recent call last): -[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in -[rank5]: pretrain( -[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain -[rank5]: iteration, num_floating_point_operations_so_far = train( -[rank5]: ^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train -[rank5]: ) = train_step( -[rank5]: ^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step -[rank5]: losses_reduced = forward_backward_func( -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank5]: output_tensor, num_tokens = forward_step( -[rank5]: ^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank5]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step -[rank5]: output_tensor = model( -[rank5]: ^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank5]: return forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank5]: return self.module(*inputs, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank5]: return forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward -[rank5]: outputs = self.module(*inputs, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank5]: return forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank5]: hidden_states = self.decoder( -[rank5]: ^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank5]: return forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank5]: hidden_states, context = layer( -[rank5]: ^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ -[rank5]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank5]: return forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank5]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp -[rank5]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank5]: return forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward -[rank5]: return self.ultraep_forward(hidden_states) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward -[rank5]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute -[rank5]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute -[rank5]: expanded_probs, expanded_routing_map = self._dense_reroute( -[rank5]: ^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute -[rank5]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank5]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward -[rank5]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -[rank6]: Traceback (most recent call last): -[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in -[rank6]: pretrain( -[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain -[rank6]: iteration, num_floating_point_operations_so_far = train( -[rank6]: ^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train -[rank6]: ) = train_step( -[rank6]: ^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step -[rank6]: losses_reduced = forward_backward_func( -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank6]: output_tensor, num_tokens = forward_step( -[rank6]: ^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step -[rank6]: output_tensor = model( -[rank6]: ^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank6]: return forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank6]: return self.module(*inputs, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank6]: return forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward -[rank6]: outputs = self.module(*inputs, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank6]: return forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank6]: hidden_states = self.decoder( -[rank6]: ^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank6]: return forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank6]: hidden_states, context = layer( -[rank6]: ^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ -[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank6]: return forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp -[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank6]: return forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward -[rank6]: return self.ultraep_forward(hidden_states) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward -[rank6]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute -[rank6]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute -[rank6]: expanded_probs, expanded_routing_map = self._dense_reroute( -[rank6]: ^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute -[rank6]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward -[rank6]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -[rank0]: Traceback (most recent call last): -[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 394, in -[rank0]: pretrain( -[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 897, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: ^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 2319, in train -[rank0]: ) = train_step( -[rank0]: ^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/training/training.py", line 1413, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: ^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/pretrain_gpt.py", line 315, in forward_step -[rank0]: output_tensor = model( -[rank0]: ^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/module.py", line 237, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank0]: hidden_states = self.decoder( -[rank0]: ^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank0]: hidden_states, context = layer( -[rank0]: ^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 880, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank0]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_layer.py", line 625, in _forward_mlp -[rank0]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 871, in forward -[rank0]: return self.ultraep_forward(hidden_states) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/moe_layer.py", line 569, in ultraep_forward -[rank0]: probs, routing_map = mgr.reroute(virtual_layer_id, probs, routing_map) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/Megatron-UltraEP/megatron/core/transformer/moe/ultraep_manager.py", line 178, in reroute -[rank0]: return self.runtime.reroute(virtual_layer_id, probs, routing_map) -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/UltraEP/ultra_ep/manager.py", line 603, in reroute -[rank0]: expanded_probs, expanded_routing_map = self._dense_reroute( -[rank0]: ^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/UltraEP/ultra_ep/manager.py", line 621, in _dense_reroute -[rank0]: expanded_probs, expanded_routing_map = _DenseRerouteFunction.apply( -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: File "/workspace/UltraEP/ultra_ep/reroute.py", line 17, in forward -[rank0]: expanded_probs, expanded_routing_map = manager_runtime.dense_reroute_forward( -[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank0]: RuntimeError: Assertion exception (/workspace/UltraEP/csrc/ultra_ep.cpp:909): probs.scalar_type() == torch::kFloat32 -[rank0]:[W826 15:05:13.484711782 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 15:05:14.558000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60311 closing signal SIGTERM -W0826 15:05:14.559000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60313 closing signal SIGTERM -W0826 15:05:14.559000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60314 closing signal SIGTERM -W0826 15:05:14.559000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60315 closing signal SIGTERM -W0826 15:05:14.559000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60316 closing signal SIGTERM -W0826 15:05:14.560000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60317 closing signal SIGTERM -W0826 15:05:14.560000 60221 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 60318 closing signal SIGTERM -E0826 15:05:16.648000 60221 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 1 (pid: 60312) of binary: /usr/bin/python +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3296.29, 3302.59) + train/valid/test-data-iterators-setup ..........: (66.87, 85.86) +[before the start of training step] datetime: 2026-08-26 15:51:30 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=30836 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=30836 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=30584 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=30584 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31104 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=31902 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=31902 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32238 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32238 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32249 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32342 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32249 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32342 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=31518 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32012 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32116 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=31518 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32012 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32116 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32320 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=31981 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32368 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=31981 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32368 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=31757 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=31757 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31123 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=31390 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31123 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=31390 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32230 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32230 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32195 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32195 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=10 total_expected_recv=328324 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=10 total_expected_recv=327892 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76160 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=10 total_expected_recv=326247 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=10 total_expected_recv=323680 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79616 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=10 total_expected_recv=329251 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74624 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=10 total_expected_recv=326951 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=10 total_expected_recv=329903 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74368 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=10 total_expected_recv=329192 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74752 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:51:58] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 27791.0 | throughput per GPU (TFLOP/s/GPU): 4.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097922E+01 | loss scale: 1.0 | grad norm: 29.871 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 64253.6591796875 | max allocated: 66328.82763671875 | reserved: 73056.0 | max reserved: 73056.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33073 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32059 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33073 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32175 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32455 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32059 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32175 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=31592 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=31975 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=31592 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=31975 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=31699 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=31869 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=31699 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=31869 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=31702 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=31702 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=31923 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=31923 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32194 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32194 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=31869 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=31869 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32084 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31379 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31379 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33099 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32190 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33099 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32190 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32116 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=31453 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32116 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=31453 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32015 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32015 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=31396 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32173 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=31396 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32173 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=10 total_expected_recv=328683 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74752 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=10 total_expected_recv=326556 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=10 total_expected_recv=324921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78720 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=10 total_expected_recv=326390 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=10 total_expected_recv=328311 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75392 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=10 total_expected_recv=329947 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=73984 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=10 total_expected_recv=328404 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74880 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=10 total_expected_recv=328228 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75392 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:51:58] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 402.2 | throughput per GPU (TFLOP/s/GPU): 307.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097874E+01 | loss scale: 1.0 | grad norm: 30.625 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33792 | min safe capacity: 1.0312 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31588 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32004 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31588 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32004 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=31413 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=31413 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32354 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32354 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=30717 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=30717 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=31385 expected_overflow=0 exact_recv_per_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=31385 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32182 expected_overflow=0 exact_recv_per_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32182 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32263 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=31305 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32263 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=31305 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=31002 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=31002 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=31309 expected_overflow=0 exact_recv_per_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=31309 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31104 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=31638 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=31931 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=31638 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=31931 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[19424, 34223, 14850, 34966, 30877, 33072, 67427, 27305] avg=32768.0000 max=67427 max_rank=6 min=14850 min_rank=2 imbalance_ratio=2.057709 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[19424, 34223, 14850, 34966, 30877, 33072, 67427, 27305] avg=32768.0000 max=67427 max_rank=6 min=14850 min_rank=2 imbalance_ratio=2.057709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[19424, 34223, 14850, 34966, 30877, 33072, 67427, 27305] avg=32768.0000 max=67427 max_rank=6 min=14850 min_rank=2 imbalance_ratio=2.057709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[19424, 34223, 14850, 34966, 30877, 33072, 67427, 27305] avg=32768.0000 max=67427 max_rank=6 min=14850 min_rank=2 imbalance_ratio=2.057709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[19424, 34223, 14850, 34966, 30877, 33072, 67427, 27305] avg=32768.0000 max=67427 max_rank=6 min=14850 min_rank=2 imbalance_ratio=2.057709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[19424, 34223, 14850, 34966, 30877, 33072, 67427, 27305] avg=32768.0000 max=67427 max_rank=6 min=14850 min_rank=2 imbalance_ratio=2.057709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] avg=32768.0000 max=33096 max_rank=0 min=31588 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34331 max_load_reduction_pct=0.509158 moved_to_replicas=37328 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] avg=32768.0000 max=33096 max_rank=0 min=31588 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34331 max_load_reduction_pct=0.509158 moved_to_replicas=37328 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] avg=32768.0000 max=33096 max_rank=0 min=31588 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34331 max_load_reduction_pct=0.509158 moved_to_replicas=37328 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[19424, 34223, 14850, 34966, 30877, 33072, 67427, 27305] avg=32768.0000 max=67427 max_rank=6 min=14850 min_rank=2 imbalance_ratio=2.057709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] avg=32768.0000 max=33096 max_rank=0 min=31588 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34331 max_load_reduction_pct=0.509158 moved_to_replicas=37328 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] avg=32768.0000 max=33096 max_rank=0 min=31588 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34331 max_load_reduction_pct=0.509158 moved_to_replicas=37328 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[19424, 34223, 14850, 34966, 30877, 33072, 67427, 27305] avg=32768.0000 max=67427 max_rank=6 min=14850 min_rank=2 imbalance_ratio=2.057709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] avg=32768.0000 max=33096 max_rank=0 min=31588 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34331 max_load_reduction_pct=0.509158 moved_to_replicas=37328 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] avg=32768.0000 max=33096 max_rank=0 min=31588 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34331 max_load_reduction_pct=0.509158 moved_to_replicas=37328 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[17475, 68754, 15200, 27355, 32897, 14688, 42106, 43669] avg=32768.0000 max=68754 max_rank=1 min=14688 min_rank=5 imbalance_ratio=2.098206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[17475, 68754, 15200, 27355, 32897, 14688, 42106, 43669] avg=32768.0000 max=68754 max_rank=1 min=14688 min_rank=5 imbalance_ratio=2.098206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[17475, 68754, 15200, 27355, 32897, 14688, 42106, 43669] avg=32768.0000 max=68754 max_rank=1 min=14688 min_rank=5 imbalance_ratio=2.098206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32004, 33072, 33096, 31588] avg=32768.0000 max=33096 max_rank=0 min=31588 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34331 max_load_reduction_pct=0.509158 moved_to_replicas=37328 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[17475, 68754, 15200, 27355, 32897, 14688, 42106, 43669] avg=32768.0000 max=68754 max_rank=1 min=14688 min_rank=5 imbalance_ratio=2.098206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[17475, 68754, 15200, 27355, 32897, 14688, 42106, 43669] avg=32768.0000 max=68754 max_rank=1 min=14688 min_rank=5 imbalance_ratio=2.098206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[17475, 68754, 15200, 27355, 32897, 14688, 42106, 43669] avg=32768.0000 max=68754 max_rank=1 min=14688 min_rank=5 imbalance_ratio=2.098206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31413 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35658 max_load_reduction_pct=0.518632 moved_to_replicas=55241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31413 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35658 max_load_reduction_pct=0.518632 moved_to_replicas=55241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31413 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35658 max_load_reduction_pct=0.518632 moved_to_replicas=55241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31413 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35658 max_load_reduction_pct=0.518632 moved_to_replicas=55241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[17475, 68754, 15200, 27355, 32897, 14688, 42106, 43669] avg=32768.0000 max=68754 max_rank=1 min=14688 min_rank=5 imbalance_ratio=2.098206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31413 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35658 max_load_reduction_pct=0.518632 moved_to_replicas=55241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31413 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35658 max_load_reduction_pct=0.518632 moved_to_replicas=55241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[17475, 68754, 15200, 27355, 32897, 14688, 42106, 43669] avg=32768.0000 max=68754 max_rank=1 min=14688 min_rank=5 imbalance_ratio=2.098206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[25023, 49445, 24098, 38045, 35329, 30618, 22261, 37325] avg=32768.0000 max=49445 max_rank=1 min=22261 min_rank=6 imbalance_ratio=1.508942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[25023, 49445, 24098, 38045, 35329, 30618, 22261, 37325] avg=32768.0000 max=49445 max_rank=1 min=22261 min_rank=6 imbalance_ratio=1.508942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[25023, 49445, 24098, 38045, 35329, 30618, 22261, 37325] avg=32768.0000 max=49445 max_rank=1 min=22261 min_rank=6 imbalance_ratio=1.508942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31413 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35658 max_load_reduction_pct=0.518632 moved_to_replicas=55241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[33096, 33096, 31413, 33096, 32897, 32354, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31413 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35658 max_load_reduction_pct=0.518632 moved_to_replicas=55241 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[25023, 49445, 24098, 38045, 35329, 30618, 22261, 37325] avg=32768.0000 max=49445 max_rank=1 min=22261 min_rank=6 imbalance_ratio=1.508942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=16348 max_load_reduction_pct=0.330630 moved_to_replicas=27760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[25023, 49445, 24098, 38045, 35329, 30618, 22261, 37325] avg=32768.0000 max=49445 max_rank=1 min=22261 min_rank=6 imbalance_ratio=1.508942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=16348 max_load_reduction_pct=0.330630 moved_to_replicas=27760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=16348 max_load_reduction_pct=0.330630 moved_to_replicas=27760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[25023, 49445, 24098, 38045, 35329, 30618, 22261, 37325] avg=32768.0000 max=49445 max_rank=1 min=22261 min_rank=6 imbalance_ratio=1.508942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=16348 max_load_reduction_pct=0.330630 moved_to_replicas=27760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[25023, 49445, 24098, 38045, 35329, 30618, 22261, 37325] avg=32768.0000 max=49445 max_rank=1 min=22261 min_rank=6 imbalance_ratio=1.508942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[25023, 49445, 24098, 38045, 35329, 30618, 22261, 37325] avg=32768.0000 max=49445 max_rank=1 min=22261 min_rank=6 imbalance_ratio=1.508942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=16348 max_load_reduction_pct=0.330630 moved_to_replicas=27760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[44854, 52798, 23143, 16980, 48510, 34725, 22487, 18647] avg=32768.0000 max=52798 max_rank=1 min=16980 min_rank=3 imbalance_ratio=1.611267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=16348 max_load_reduction_pct=0.330630 moved_to_replicas=27760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[44854, 52798, 23143, 16980, 48510, 34725, 22487, 18647] avg=32768.0000 max=52798 max_rank=1 min=16980 min_rank=3 imbalance_ratio=1.611267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[44854, 52798, 23143, 16980, 48510, 34725, 22487, 18647] avg=32768.0000 max=52798 max_rank=1 min=16980 min_rank=3 imbalance_ratio=1.611267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=16348 max_load_reduction_pct=0.330630 moved_to_replicas=27760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[30717, 33096, 33095, 33096, 33097, 32850, 33097, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=16348 max_load_reduction_pct=0.330630 moved_to_replicas=27760 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[44854, 52798, 23143, 16980, 48510, 34725, 22487, 18647] avg=32768.0000 max=52798 max_rank=1 min=16980 min_rank=3 imbalance_ratio=1.611267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19701 max_load_reduction_pct=0.373139 moved_to_replicas=48503 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19701 max_load_reduction_pct=0.373139 moved_to_replicas=48503 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19701 max_load_reduction_pct=0.373139 moved_to_replicas=48503 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[44854, 52798, 23143, 16980, 48510, 34725, 22487, 18647] avg=32768.0000 max=52798 max_rank=1 min=16980 min_rank=3 imbalance_ratio=1.611267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[44854, 52798, 23143, 16980, 48510, 34725, 22487, 18647] avg=32768.0000 max=52798 max_rank=1 min=16980 min_rank=3 imbalance_ratio=1.611267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19701 max_load_reduction_pct=0.373139 moved_to_replicas=48503 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[44854, 52798, 23143, 16980, 48510, 34725, 22487, 18647] avg=32768.0000 max=52798 max_rank=1 min=16980 min_rank=3 imbalance_ratio=1.611267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[44854, 52798, 23143, 16980, 48510, 34725, 22487, 18647] avg=32768.0000 max=52798 max_rank=1 min=16980 min_rank=3 imbalance_ratio=1.611267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[24000, 8058, 67163, 9720, 37788, 35999, 31385, 48031] avg=32768.0000 max=67163 max_rank=2 min=8058 min_rank=1 imbalance_ratio=2.049652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19701 max_load_reduction_pct=0.373139 moved_to_replicas=48503 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19701 max_load_reduction_pct=0.373139 moved_to_replicas=48503 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[24000, 8058, 67163, 9720, 37788, 35999, 31385, 48031] avg=32768.0000 max=67163 max_rank=2 min=8058 min_rank=1 imbalance_ratio=2.049652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[24000, 8058, 67163, 9720, 37788, 35999, 31385, 48031] avg=32768.0000 max=67163 max_rank=2 min=8058 min_rank=1 imbalance_ratio=2.049652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19701 max_load_reduction_pct=0.373139 moved_to_replicas=48503 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33095, 33096, 30472, 33097] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19701 max_load_reduction_pct=0.373139 moved_to_replicas=48503 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[24000, 8058, 67163, 9720, 37788, 35999, 31385, 48031] avg=32768.0000 max=67163 max_rank=2 min=8058 min_rank=1 imbalance_ratio=2.049652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] avg=32768.0000 max=33097 max_rank=3 min=31385 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34066 max_load_reduction_pct=0.507214 moved_to_replicas=56597 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] avg=32768.0000 max=33097 max_rank=3 min=31385 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34066 max_load_reduction_pct=0.507214 moved_to_replicas=56597 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] avg=32768.0000 max=33097 max_rank=3 min=31385 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34066 max_load_reduction_pct=0.507214 moved_to_replicas=56597 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[24000, 8058, 67163, 9720, 37788, 35999, 31385, 48031] avg=32768.0000 max=67163 max_rank=2 min=8058 min_rank=1 imbalance_ratio=2.049652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[24000, 8058, 67163, 9720, 37788, 35999, 31385, 48031] avg=32768.0000 max=67163 max_rank=2 min=8058 min_rank=1 imbalance_ratio=2.049652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] avg=32768.0000 max=33097 max_rank=3 min=31385 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34066 max_load_reduction_pct=0.507214 moved_to_replicas=56597 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[24000, 8058, 67163, 9720, 37788, 35999, 31385, 48031] avg=32768.0000 max=67163 max_rank=2 min=8058 min_rank=1 imbalance_ratio=2.049652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[24000, 8058, 67163, 9720, 37788, 35999, 31385, 48031] avg=32768.0000 max=67163 max_rank=2 min=8058 min_rank=1 imbalance_ratio=2.049652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] avg=32768.0000 max=33097 max_rank=3 min=31385 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34066 max_load_reduction_pct=0.507214 moved_to_replicas=56597 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[47050, 53490, 32263, 24771, 36946, 18900, 34769, 13955] avg=32768.0000 max=53490 max_rank=1 min=13955 min_rank=7 imbalance_ratio=1.632385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] avg=32768.0000 max=33097 max_rank=3 min=31385 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34066 max_load_reduction_pct=0.507214 moved_to_replicas=56597 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[47050, 53490, 32263, 24771, 36946, 18900, 34769, 13955] avg=32768.0000 max=53490 max_rank=1 min=13955 min_rank=7 imbalance_ratio=1.632385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[47050, 53490, 32263, 24771, 36946, 18900, 34769, 13955] avg=32768.0000 max=53490 max_rank=1 min=13955 min_rank=7 imbalance_ratio=1.632385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] avg=32768.0000 max=33097 max_rank=3 min=31385 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34066 max_load_reduction_pct=0.507214 moved_to_replicas=56597 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32182, 33096, 33096, 33097, 33095, 33096, 31385, 33097] avg=32768.0000 max=33097 max_rank=3 min=31385 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34066 max_load_reduction_pct=0.507214 moved_to_replicas=56597 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31305 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20393 max_load_reduction_pct=0.381249 moved_to_replicas=39871 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[47050, 53490, 32263, 24771, 36946, 18900, 34769, 13955] avg=32768.0000 max=53490 max_rank=1 min=13955 min_rank=7 imbalance_ratio=1.632385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31305 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20393 max_load_reduction_pct=0.381249 moved_to_replicas=39871 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31305 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20393 max_load_reduction_pct=0.381249 moved_to_replicas=39871 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[47050, 53490, 32263, 24771, 36946, 18900, 34769, 13955] avg=32768.0000 max=53490 max_rank=1 min=13955 min_rank=7 imbalance_ratio=1.632385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[47050, 53490, 32263, 24771, 36946, 18900, 34769, 13955] avg=32768.0000 max=53490 max_rank=1 min=13955 min_rank=7 imbalance_ratio=1.632385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31305 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20393 max_load_reduction_pct=0.381249 moved_to_replicas=39871 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[47050, 53490, 32263, 24771, 36946, 18900, 34769, 13955] avg=32768.0000 max=53490 max_rank=1 min=13955 min_rank=7 imbalance_ratio=1.632385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[47050, 53490, 32263, 24771, 36946, 18900, 34769, 13955] avg=32768.0000 max=53490 max_rank=1 min=13955 min_rank=7 imbalance_ratio=1.632385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[44564, 23083, 32566, 58495, 4646, 45730, 12293, 40767] avg=32768.0000 max=58495 max_rank=3 min=4646 min_rank=4 imbalance_ratio=1.785126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31305 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20393 max_load_reduction_pct=0.381249 moved_to_replicas=39871 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31305 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20393 max_load_reduction_pct=0.381249 moved_to_replicas=39871 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[44564, 23083, 32566, 58495, 4646, 45730, 12293, 40767] avg=32768.0000 max=58495 max_rank=3 min=4646 min_rank=4 imbalance_ratio=1.785126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31305 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20393 max_load_reduction_pct=0.381249 moved_to_replicas=39871 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[44564, 23083, 32566, 58495, 4646, 45730, 12293, 40767] avg=32768.0000 max=58495 max_rank=3 min=4646 min_rank=4 imbalance_ratio=1.785126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33095, 32263, 31305, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31305 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20393 max_load_reduction_pct=0.381249 moved_to_replicas=39871 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=31002 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25398 max_load_reduction_pct=0.434191 moved_to_replicas=57172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[44564, 23083, 32566, 58495, 4646, 45730, 12293, 40767] avg=32768.0000 max=58495 max_rank=3 min=4646 min_rank=4 imbalance_ratio=1.785126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=31002 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25398 max_load_reduction_pct=0.434191 moved_to_replicas=57172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=31002 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25398 max_load_reduction_pct=0.434191 moved_to_replicas=57172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[44564, 23083, 32566, 58495, 4646, 45730, 12293, 40767] avg=32768.0000 max=58495 max_rank=3 min=4646 min_rank=4 imbalance_ratio=1.785126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[44564, 23083, 32566, 58495, 4646, 45730, 12293, 40767] avg=32768.0000 max=58495 max_rank=3 min=4646 min_rank=4 imbalance_ratio=1.785126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[44564, 23083, 32566, 58495, 4646, 45730, 12293, 40767] avg=32768.0000 max=58495 max_rank=3 min=4646 min_rank=4 imbalance_ratio=1.785126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=31002 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25398 max_load_reduction_pct=0.434191 moved_to_replicas=57172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[44564, 23083, 32566, 58495, 4646, 45730, 12293, 40767] avg=32768.0000 max=58495 max_rank=3 min=4646 min_rank=4 imbalance_ratio=1.785126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[27684, 38409, 23127, 75830, 13018, 25093, 38046, 20937] avg=32768.0000 max=75830 max_rank=3 min=13018 min_rank=4 imbalance_ratio=2.314148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=31002 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25398 max_load_reduction_pct=0.434191 moved_to_replicas=57172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=31002 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25398 max_load_reduction_pct=0.434191 moved_to_replicas=57172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[27684, 38409, 23127, 75830, 13018, 25093, 38046, 20937] avg=32768.0000 max=75830 max_rank=3 min=13018 min_rank=4 imbalance_ratio=2.314148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=31002 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25398 max_load_reduction_pct=0.434191 moved_to_replicas=57172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[27684, 38409, 23127, 75830, 13018, 25093, 38046, 20937] avg=32768.0000 max=75830 max_rank=3 min=13018 min_rank=4 imbalance_ratio=2.314148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[33095, 33097, 32566, 33096, 31002, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=31002 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25398 max_load_reduction_pct=0.434191 moved_to_replicas=57172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] avg=32768.0000 max=33097 max_rank=3 min=31309 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42733 max_load_reduction_pct=0.563537 moved_to_replicas=52997 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[27684, 38409, 23127, 75830, 13018, 25093, 38046, 20937] avg=32768.0000 max=75830 max_rank=3 min=13018 min_rank=4 imbalance_ratio=2.314148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] avg=32768.0000 max=33097 max_rank=3 min=31309 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42733 max_load_reduction_pct=0.563537 moved_to_replicas=52997 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] avg=32768.0000 max=33097 max_rank=3 min=31309 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42733 max_load_reduction_pct=0.563537 moved_to_replicas=52997 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[27684, 38409, 23127, 75830, 13018, 25093, 38046, 20937] avg=32768.0000 max=75830 max_rank=3 min=13018 min_rank=4 imbalance_ratio=2.314148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[27684, 38409, 23127, 75830, 13018, 25093, 38046, 20937] avg=32768.0000 max=75830 max_rank=3 min=13018 min_rank=4 imbalance_ratio=2.314148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[27684, 38409, 23127, 75830, 13018, 25093, 38046, 20937] avg=32768.0000 max=75830 max_rank=3 min=13018 min_rank=4 imbalance_ratio=2.314148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] avg=32768.0000 max=33097 max_rank=3 min=31309 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42733 max_load_reduction_pct=0.563537 moved_to_replicas=52997 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[34639, 39882, 33311, 33558, 13503, 21676, 58199, 27376] avg=32768.0000 max=58199 max_rank=6 min=13503 min_rank=4 imbalance_ratio=1.776093 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[27684, 38409, 23127, 75830, 13018, 25093, 38046, 20937] avg=32768.0000 max=75830 max_rank=3 min=13018 min_rank=4 imbalance_ratio=2.314148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] avg=32768.0000 max=33097 max_rank=3 min=31309 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42733 max_load_reduction_pct=0.563537 moved_to_replicas=52997 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] avg=32768.0000 max=33097 max_rank=3 min=31309 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42733 max_load_reduction_pct=0.563537 moved_to_replicas=52997 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[34639, 39882, 33311, 33558, 13503, 21676, 58199, 27376] avg=32768.0000 max=58199 max_rank=6 min=13503 min_rank=4 imbalance_ratio=1.776093 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] avg=32768.0000 max=33097 max_rank=3 min=31309 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42733 max_load_reduction_pct=0.563537 moved_to_replicas=52997 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[34639, 39882, 33311, 33558, 13503, 21676, 58199, 27376] avg=32768.0000 max=58199 max_rank=6 min=13503 min_rank=4 imbalance_ratio=1.776093 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25103 max_load_reduction_pct=0.431330 moved_to_replicas=34109 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[32634, 33096, 33095, 33097, 33095, 32722, 33096, 31309] avg=32768.0000 max=33097 max_rank=3 min=31309 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42733 max_load_reduction_pct=0.563537 moved_to_replicas=52997 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[34639, 39882, 33311, 33558, 13503, 21676, 58199, 27376] avg=32768.0000 max=58199 max_rank=6 min=13503 min_rank=4 imbalance_ratio=1.776093 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25103 max_load_reduction_pct=0.431330 moved_to_replicas=34109 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25103 max_load_reduction_pct=0.431330 moved_to_replicas=34109 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[34639, 39882, 33311, 33558, 13503, 21676, 58199, 27376] avg=32768.0000 max=58199 max_rank=6 min=13503 min_rank=4 imbalance_ratio=1.776093 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[34639, 39882, 33311, 33558, 13503, 21676, 58199, 27376] avg=32768.0000 max=58199 max_rank=6 min=13503 min_rank=4 imbalance_ratio=1.776093 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[34639, 39882, 33311, 33558, 13503, 21676, 58199, 27376] avg=32768.0000 max=58199 max_rank=6 min=13503 min_rank=4 imbalance_ratio=1.776093 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25103 max_load_reduction_pct=0.431330 moved_to_replicas=34109 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[14893, 13759, 3312, 46005, 41703, 52965, 17355, 72152] avg=32768.0000 max=72152 max_rank=7 min=3312 min_rank=2 imbalance_ratio=2.201904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[34639, 39882, 33311, 33558, 13503, 21676, 58199, 27376] avg=32768.0000 max=58199 max_rank=6 min=13503 min_rank=4 imbalance_ratio=1.776093 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[14893, 13759, 3312, 46005, 41703, 52965, 17355, 72152] avg=32768.0000 max=72152 max_rank=7 min=3312 min_rank=2 imbalance_ratio=2.201904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25103 max_load_reduction_pct=0.431330 moved_to_replicas=34109 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25103 max_load_reduction_pct=0.431330 moved_to_replicas=34109 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25103 max_load_reduction_pct=0.431330 moved_to_replicas=34109 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[14893, 13759, 3312, 46005, 41703, 52965, 17355, 72152] avg=32768.0000 max=72152 max_rank=7 min=3312 min_rank=2 imbalance_ratio=2.201904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] avg=32768.0000 max=33097 max_rank=4 min=31638 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39055 max_load_reduction_pct=0.541288 moved_to_replicas=80441 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25103 max_load_reduction_pct=0.431330 moved_to_replicas=34109 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] avg=32768.0000 max=33097 max_rank=4 min=31638 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39055 max_load_reduction_pct=0.541288 moved_to_replicas=80441 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[14893, 13759, 3312, 46005, 41703, 52965, 17355, 72152] avg=32768.0000 max=72152 max_rank=7 min=3312 min_rank=2 imbalance_ratio=2.201904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] avg=32768.0000 max=33097 max_rank=4 min=31638 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39055 max_load_reduction_pct=0.541288 moved_to_replicas=80441 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[14893, 13759, 3312, 46005, 41703, 52965, 17355, 72152] avg=32768.0000 max=72152 max_rank=7 min=3312 min_rank=2 imbalance_ratio=2.201904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[14893, 13759, 3312, 46005, 41703, 52965, 17355, 72152] avg=32768.0000 max=72152 max_rank=7 min=3312 min_rank=2 imbalance_ratio=2.201904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[14893, 13759, 3312, 46005, 41703, 52965, 17355, 72152] avg=32768.0000 max=72152 max_rank=7 min=3312 min_rank=2 imbalance_ratio=2.201904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] avg=32768.0000 max=33097 max_rank=4 min=31638 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39055 max_load_reduction_pct=0.541288 moved_to_replicas=80441 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] avg=32768.0000 max=33097 max_rank=4 min=31638 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39055 max_load_reduction_pct=0.541288 moved_to_replicas=80441 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[14893, 13759, 3312, 46005, 41703, 52965, 17355, 72152] avg=32768.0000 max=72152 max_rank=7 min=3312 min_rank=2 imbalance_ratio=2.201904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] avg=32768.0000 max=33097 max_rank=4 min=31638 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39055 max_load_reduction_pct=0.541288 moved_to_replicas=80441 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] avg=32768.0000 max=33097 max_rank=4 min=31638 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39055 max_load_reduction_pct=0.541288 moved_to_replicas=80441 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 31638, 33095, 33097, 33097, 31931, 33096] avg=32768.0000 max=33097 max_rank=4 min=31638 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39055 max_load_reduction_pct=0.541288 moved_to_replicas=80441 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=10 total_expected_recv=325044 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78848 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=10 total_expected_recv=329170 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=73984 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=10 total_expected_recv=326453 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=10 total_expected_recv=327575 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75648 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=10 total_expected_recv=327205 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=10 total_expected_recv=329573 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=73216 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=10 total_expected_recv=325461 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=10 total_expected_recv=330959 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=72192 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:51:59] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 611.3 | throughput per GPU (TFLOP/s/GPU): 202.0 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.539755E+00 | loss scale: 1.0 | grad norm: 32.424 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33792 | min safe capacity: 1.0312 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=30950 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=30950 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=30719 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=30719 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32138 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=31798 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32138 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=31798 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=30856 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=30856 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=30858 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=30858 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=31905 expected_overflow=0 exact_recv_per_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=31905 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32006 expected_overflow=0 exact_recv_per_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32006 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=31758 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=31758 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=31707 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=31707 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32195 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32195 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=31504 expected_overflow=0 exact_recv_per_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=31504 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=30896 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32064 expected_overflow=0 exact_recv_per_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=30896 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32064 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[23851, 20086, 20587, 43021, 23917, 44588, 57858, 28236] avg=32768.0000 max=57858 max_rank=6 min=20086 min_rank=1 imbalance_ratio=1.765686 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[23851, 20086, 20587, 43021, 23917, 44588, 57858, 28236] avg=32768.0000 max=57858 max_rank=6 min=20086 min_rank=1 imbalance_ratio=1.765686 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] avg=32768.0000 max=33097 max_rank=6 min=30950 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24761 max_load_reduction_pct=0.427962 moved_to_replicas=46179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[23851, 20086, 20587, 43021, 23917, 44588, 57858, 28236] avg=32768.0000 max=57858 max_rank=6 min=20086 min_rank=1 imbalance_ratio=1.765686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[23851, 20086, 20587, 43021, 23917, 44588, 57858, 28236] avg=32768.0000 max=57858 max_rank=6 min=20086 min_rank=1 imbalance_ratio=1.765686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[23851, 20086, 20587, 43021, 23917, 44588, 57858, 28236] avg=32768.0000 max=57858 max_rank=6 min=20086 min_rank=1 imbalance_ratio=1.765686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] avg=32768.0000 max=33097 max_rank=6 min=30950 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24761 max_load_reduction_pct=0.427962 moved_to_replicas=46179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[23851, 20086, 20587, 43021, 23917, 44588, 57858, 28236] avg=32768.0000 max=57858 max_rank=6 min=20086 min_rank=1 imbalance_ratio=1.765686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[23851, 20086, 20587, 43021, 23917, 44588, 57858, 28236] avg=32768.0000 max=57858 max_rank=6 min=20086 min_rank=1 imbalance_ratio=1.765686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] avg=32768.0000 max=33097 max_rank=6 min=30950 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24761 max_load_reduction_pct=0.427962 moved_to_replicas=46179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] avg=32768.0000 max=33097 max_rank=6 min=30950 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24761 max_load_reduction_pct=0.427962 moved_to_replicas=46179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] avg=32768.0000 max=33097 max_rank=6 min=30950 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24761 max_load_reduction_pct=0.427962 moved_to_replicas=46179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[23851, 20086, 20587, 43021, 23917, 44588, 57858, 28236] avg=32768.0000 max=57858 max_rank=6 min=20086 min_rank=1 imbalance_ratio=1.765686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] avg=32768.0000 max=33097 max_rank=6 min=30950 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24761 max_load_reduction_pct=0.427962 moved_to_replicas=46179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[63798, 25135, 19579, 32849, 19694, 30325, 25701, 45063] avg=32768.0000 max=63798 max_rank=0 min=19579 min_rank=2 imbalance_ratio=1.946960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[63798, 25135, 19579, 32849, 19694, 30325, 25701, 45063] avg=32768.0000 max=63798 max_rank=0 min=19579 min_rank=2 imbalance_ratio=1.946960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] avg=32768.0000 max=33097 max_rank=6 min=30950 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24761 max_load_reduction_pct=0.427962 moved_to_replicas=46179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 30950, 33096, 33096, 33096, 33097, 32618] avg=32768.0000 max=33097 max_rank=6 min=30950 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24761 max_load_reduction_pct=0.427962 moved_to_replicas=46179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[63798, 25135, 19579, 32849, 19694, 30325, 25701, 45063] avg=32768.0000 max=63798 max_rank=0 min=19579 min_rank=2 imbalance_ratio=1.946960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30719 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30701 max_load_reduction_pct=0.481222 moved_to_replicas=42669 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30719 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30701 max_load_reduction_pct=0.481222 moved_to_replicas=42669 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[63798, 25135, 19579, 32849, 19694, 30325, 25701, 45063] avg=32768.0000 max=63798 max_rank=0 min=19579 min_rank=2 imbalance_ratio=1.946960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[63798, 25135, 19579, 32849, 19694, 30325, 25701, 45063] avg=32768.0000 max=63798 max_rank=0 min=19579 min_rank=2 imbalance_ratio=1.946960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[63798, 25135, 19579, 32849, 19694, 30325, 25701, 45063] avg=32768.0000 max=63798 max_rank=0 min=19579 min_rank=2 imbalance_ratio=1.946960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[63798, 25135, 19579, 32849, 19694, 30325, 25701, 45063] avg=32768.0000 max=63798 max_rank=0 min=19579 min_rank=2 imbalance_ratio=1.946960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30719 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30701 max_load_reduction_pct=0.481222 moved_to_replicas=42669 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30719 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30701 max_load_reduction_pct=0.481222 moved_to_replicas=42669 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30719 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30701 max_load_reduction_pct=0.481222 moved_to_replicas=42669 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[63798, 25135, 19579, 32849, 19694, 30325, 25701, 45063] avg=32768.0000 max=63798 max_rank=0 min=19579 min_rank=2 imbalance_ratio=1.946960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30719 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30701 max_load_reduction_pct=0.481222 moved_to_replicas=42669 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[43506, 33118, 31434, 13436, 57841, 33800, 40473, 8536] avg=32768.0000 max=57841 max_rank=4 min=8536 min_rank=7 imbalance_ratio=1.765167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[43506, 33118, 31434, 13436, 57841, 33800, 40473, 8536] avg=32768.0000 max=57841 max_rank=4 min=8536 min_rank=7 imbalance_ratio=1.765167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30719 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30701 max_load_reduction_pct=0.481222 moved_to_replicas=42669 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 32849, 33096, 30719, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30719 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30701 max_load_reduction_pct=0.481222 moved_to_replicas=42669 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] avg=32768.0000 max=33097 max_rank=6 min=31798 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=24744 max_load_reduction_pct=0.427793 moved_to_replicas=43258 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[43506, 33118, 31434, 13436, 57841, 33800, 40473, 8536] avg=32768.0000 max=57841 max_rank=4 min=8536 min_rank=7 imbalance_ratio=1.765167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] avg=32768.0000 max=33097 max_rank=6 min=31798 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=24744 max_load_reduction_pct=0.427793 moved_to_replicas=43258 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[43506, 33118, 31434, 13436, 57841, 33800, 40473, 8536] avg=32768.0000 max=57841 max_rank=4 min=8536 min_rank=7 imbalance_ratio=1.765167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[43506, 33118, 31434, 13436, 57841, 33800, 40473, 8536] avg=32768.0000 max=57841 max_rank=4 min=8536 min_rank=7 imbalance_ratio=1.765167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[43506, 33118, 31434, 13436, 57841, 33800, 40473, 8536] avg=32768.0000 max=57841 max_rank=4 min=8536 min_rank=7 imbalance_ratio=1.765167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[43506, 33118, 31434, 13436, 57841, 33800, 40473, 8536] avg=32768.0000 max=57841 max_rank=4 min=8536 min_rank=7 imbalance_ratio=1.765167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] avg=32768.0000 max=33097 max_rank=6 min=31798 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=24744 max_load_reduction_pct=0.427793 moved_to_replicas=43258 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] avg=32768.0000 max=33097 max_rank=6 min=31798 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=24744 max_load_reduction_pct=0.427793 moved_to_replicas=43258 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] avg=32768.0000 max=33097 max_rank=6 min=31798 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=24744 max_load_reduction_pct=0.427793 moved_to_replicas=43258 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[43506, 33118, 31434, 13436, 57841, 33800, 40473, 8536] avg=32768.0000 max=57841 max_rank=4 min=8536 min_rank=7 imbalance_ratio=1.765167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] avg=32768.0000 max=33097 max_rank=6 min=31798 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=24744 max_load_reduction_pct=0.427793 moved_to_replicas=43258 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[28231, 32712, 2181, 11035, 41693, 19544, 39986, 86762] avg=32768.0000 max=86762 max_rank=7 min=2181 min_rank=2 imbalance_ratio=2.647766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[28231, 32712, 2181, 11035, 41693, 19544, 39986, 86762] avg=32768.0000 max=86762 max_rank=7 min=2181 min_rank=2 imbalance_ratio=2.647766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] avg=32768.0000 max=33097 max_rank=6 min=31798 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=24744 max_load_reduction_pct=0.427793 moved_to_replicas=43258 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[33096, 33096, 32138, 31798, 33096, 33096, 33097, 32727] avg=32768.0000 max=33097 max_rank=6 min=31798 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=24744 max_load_reduction_pct=0.427793 moved_to_replicas=43258 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[28231, 32712, 2181, 11035, 41693, 19544, 39986, 86762] avg=32768.0000 max=86762 max_rank=7 min=2181 min_rank=2 imbalance_ratio=2.647766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[28231, 32712, 2181, 11035, 41693, 19544, 39986, 86762] avg=32768.0000 max=86762 max_rank=7 min=2181 min_rank=2 imbalance_ratio=2.647766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30856 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=53665 max_load_reduction_pct=0.618531 moved_to_replicas=69153 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30856 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=53665 max_load_reduction_pct=0.618531 moved_to_replicas=69153 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[28231, 32712, 2181, 11035, 41693, 19544, 39986, 86762] avg=32768.0000 max=86762 max_rank=7 min=2181 min_rank=2 imbalance_ratio=2.647766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[28231, 32712, 2181, 11035, 41693, 19544, 39986, 86762] avg=32768.0000 max=86762 max_rank=7 min=2181 min_rank=2 imbalance_ratio=2.647766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[28231, 32712, 2181, 11035, 41693, 19544, 39986, 86762] avg=32768.0000 max=86762 max_rank=7 min=2181 min_rank=2 imbalance_ratio=2.647766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30856 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=53665 max_load_reduction_pct=0.618531 moved_to_replicas=69153 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30856 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=53665 max_load_reduction_pct=0.618531 moved_to_replicas=69153 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30856 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=53665 max_load_reduction_pct=0.618531 moved_to_replicas=69153 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[28231, 32712, 2181, 11035, 41693, 19544, 39986, 86762] avg=32768.0000 max=86762 max_rank=7 min=2181 min_rank=2 imbalance_ratio=2.647766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30856 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=53665 max_load_reduction_pct=0.618531 moved_to_replicas=69153 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[45299, 9782, 98343, 22929, 39410, 13107, 55, 33219] avg=32768.0000 max=98343 max_rank=2 min=55 min_rank=6 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[45299, 9782, 98343, 22929, 39410, 13107, 55, 33219] avg=32768.0000 max=98343 max_rank=2 min=55 min_rank=6 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30856 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=53665 max_load_reduction_pct=0.618531 moved_to_replicas=69153 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32712, 33096, 33095, 33096, 30856, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30856 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=53665 max_load_reduction_pct=0.618531 moved_to_replicas=69153 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[45299, 9782, 98343, 22929, 39410, 13107, 55, 33219] avg=32768.0000 max=98343 max_rank=2 min=55 min_rank=6 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[45299, 9782, 98343, 22929, 39410, 13107, 55, 33219] avg=32768.0000 max=98343 max_rank=2 min=55 min_rank=6 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] avg=32768.0000 max=33097 max_rank=5 min=30858 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65246 max_load_reduction_pct=0.663453 moved_to_replicas=83887 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] avg=32768.0000 max=33097 max_rank=5 min=30858 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65246 max_load_reduction_pct=0.663453 moved_to_replicas=83887 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[45299, 9782, 98343, 22929, 39410, 13107, 55, 33219] avg=32768.0000 max=98343 max_rank=2 min=55 min_rank=6 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[45299, 9782, 98343, 22929, 39410, 13107, 55, 33219] avg=32768.0000 max=98343 max_rank=2 min=55 min_rank=6 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[45299, 9782, 98343, 22929, 39410, 13107, 55, 33219] avg=32768.0000 max=98343 max_rank=2 min=55 min_rank=6 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] avg=32768.0000 max=33097 max_rank=5 min=30858 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65246 max_load_reduction_pct=0.663453 moved_to_replicas=83887 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] avg=32768.0000 max=33097 max_rank=5 min=30858 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65246 max_load_reduction_pct=0.663453 moved_to_replicas=83887 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] avg=32768.0000 max=33097 max_rank=5 min=30858 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65246 max_load_reduction_pct=0.663453 moved_to_replicas=83887 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[45299, 9782, 98343, 22929, 39410, 13107, 55, 33219] avg=32768.0000 max=98343 max_rank=2 min=55 min_rank=6 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] avg=32768.0000 max=33097 max_rank=5 min=30858 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65246 max_load_reduction_pct=0.663453 moved_to_replicas=83887 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32006, 66615, 29749, 22931, 66553, 32753, 11245, 292] avg=32768.0000 max=66615 max_rank=1 min=292 min_rank=7 imbalance_ratio=2.032928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32006, 66615, 29749, 22931, 66553, 32753, 11245, 292] avg=32768.0000 max=66615 max_rank=1 min=292 min_rank=7 imbalance_ratio=2.032928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] avg=32768.0000 max=33097 max_rank=5 min=30858 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65246 max_load_reduction_pct=0.663453 moved_to_replicas=83887 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33519 max_load_reduction_pct=0.503175 moved_to_replicas=66976 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32006, 66615, 29749, 22931, 66553, 32753, 11245, 292] avg=32768.0000 max=66615 max_rank=1 min=292 min_rank=7 imbalance_ratio=2.032928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30858, 33096, 33097, 32710, 33096] avg=32768.0000 max=33097 max_rank=5 min=30858 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65246 max_load_reduction_pct=0.663453 moved_to_replicas=83887 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33519 max_load_reduction_pct=0.503175 moved_to_replicas=66976 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32006, 66615, 29749, 22931, 66553, 32753, 11245, 292] avg=32768.0000 max=66615 max_rank=1 min=292 min_rank=7 imbalance_ratio=2.032928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32006, 66615, 29749, 22931, 66553, 32753, 11245, 292] avg=32768.0000 max=66615 max_rank=1 min=292 min_rank=7 imbalance_ratio=2.032928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32006, 66615, 29749, 22931, 66553, 32753, 11245, 292] avg=32768.0000 max=66615 max_rank=1 min=292 min_rank=7 imbalance_ratio=2.032928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33519 max_load_reduction_pct=0.503175 moved_to_replicas=66976 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32006, 66615, 29749, 22931, 66553, 32753, 11245, 292] avg=32768.0000 max=66615 max_rank=1 min=292 min_rank=7 imbalance_ratio=2.032928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[74371, 49559, 32740, 31758, 45, 77, 73573, 21] avg=32768.0000 max=74371 max_rank=0 min=21 min_rank=7 imbalance_ratio=2.269623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33519 max_load_reduction_pct=0.503175 moved_to_replicas=66976 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33519 max_load_reduction_pct=0.503175 moved_to_replicas=66976 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33519 max_load_reduction_pct=0.503175 moved_to_replicas=66976 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32006, 66615, 29749, 22931, 66553, 32753, 11245, 292] avg=32768.0000 max=66615 max_rank=1 min=292 min_rank=7 imbalance_ratio=2.032928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[74371, 49559, 32740, 31758, 45, 77, 73573, 21] avg=32768.0000 max=74371 max_rank=0 min=21 min_rank=7 imbalance_ratio=2.269623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33519 max_load_reduction_pct=0.503175 moved_to_replicas=66976 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] avg=32768.0000 max=33097 max_rank=0 min=31758 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41274 max_load_reduction_pct=0.554974 moved_to_replicas=98215 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[74371, 49559, 32740, 31758, 45, 77, 73573, 21] avg=32768.0000 max=74371 max_rank=0 min=21 min_rank=7 imbalance_ratio=2.269623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32006, 33096, 31905, 33096, 33096, 32753, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33519 max_load_reduction_pct=0.503175 moved_to_replicas=66976 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] avg=32768.0000 max=33097 max_rank=0 min=31758 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41274 max_load_reduction_pct=0.554974 moved_to_replicas=98215 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[74371, 49559, 32740, 31758, 45, 77, 73573, 21] avg=32768.0000 max=74371 max_rank=0 min=21 min_rank=7 imbalance_ratio=2.269623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[74371, 49559, 32740, 31758, 45, 77, 73573, 21] avg=32768.0000 max=74371 max_rank=0 min=21 min_rank=7 imbalance_ratio=2.269623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[74371, 49559, 32740, 31758, 45, 77, 73573, 21] avg=32768.0000 max=74371 max_rank=0 min=21 min_rank=7 imbalance_ratio=2.269623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] avg=32768.0000 max=33097 max_rank=0 min=31758 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41274 max_load_reduction_pct=0.554974 moved_to_replicas=98215 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[74371, 49559, 32740, 31758, 45, 77, 73573, 21] avg=32768.0000 max=74371 max_rank=0 min=21 min_rank=7 imbalance_ratio=2.269623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[24413, 65254, 42492, 7107, 32762, 56646, 37, 33433] avg=32768.0000 max=65254 max_rank=1 min=37 min_rank=6 imbalance_ratio=1.991394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] avg=32768.0000 max=33097 max_rank=0 min=31758 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41274 max_load_reduction_pct=0.554974 moved_to_replicas=98215 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] avg=32768.0000 max=33097 max_rank=0 min=31758 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41274 max_load_reduction_pct=0.554974 moved_to_replicas=98215 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] avg=32768.0000 max=33097 max_rank=0 min=31758 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41274 max_load_reduction_pct=0.554974 moved_to_replicas=98215 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[24413, 65254, 42492, 7107, 32762, 56646, 37, 33433] avg=32768.0000 max=65254 max_rank=1 min=37 min_rank=6 imbalance_ratio=1.991394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[74371, 49559, 32740, 31758, 45, 77, 73573, 21] avg=32768.0000 max=74371 max_rank=0 min=21 min_rank=7 imbalance_ratio=2.269623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] avg=32768.0000 max=33097 max_rank=0 min=31758 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41274 max_load_reduction_pct=0.554974 moved_to_replicas=98215 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] avg=32768.0000 max=33096 max_rank=0 min=31707 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32158 max_load_reduction_pct=0.492813 moved_to_replicas=65441 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[24413, 65254, 42492, 7107, 32762, 56646, 37, 33433] avg=32768.0000 max=65254 max_rank=1 min=37 min_rank=6 imbalance_ratio=1.991394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] avg=32768.0000 max=33096 max_rank=0 min=31707 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32158 max_load_reduction_pct=0.492813 moved_to_replicas=65441 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32740, 31758, 33094, 32522, 33096, 32740] avg=32768.0000 max=33097 max_rank=0 min=31758 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41274 max_load_reduction_pct=0.554974 moved_to_replicas=98215 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[24413, 65254, 42492, 7107, 32762, 56646, 37, 33433] avg=32768.0000 max=65254 max_rank=1 min=37 min_rank=6 imbalance_ratio=1.991394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[24413, 65254, 42492, 7107, 32762, 56646, 37, 33433] avg=32768.0000 max=65254 max_rank=1 min=37 min_rank=6 imbalance_ratio=1.991394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[24413, 65254, 42492, 7107, 32762, 56646, 37, 33433] avg=32768.0000 max=65254 max_rank=1 min=37 min_rank=6 imbalance_ratio=1.991394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] avg=32768.0000 max=33096 max_rank=0 min=31707 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32158 max_load_reduction_pct=0.492813 moved_to_replicas=65441 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[24413, 65254, 42492, 7107, 32762, 56646, 37, 33433] avg=32768.0000 max=65254 max_rank=1 min=37 min_rank=6 imbalance_ratio=1.991394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[30364, 38496, 33985, 45773, 15030, 31504, 27097, 39895] avg=32768.0000 max=45773 max_rank=3 min=15030 min_rank=4 imbalance_ratio=1.396881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] avg=32768.0000 max=33096 max_rank=0 min=31707 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32158 max_load_reduction_pct=0.492813 moved_to_replicas=65441 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] avg=32768.0000 max=33096 max_rank=0 min=31707 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32158 max_load_reduction_pct=0.492813 moved_to_replicas=65441 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] avg=32768.0000 max=33096 max_rank=0 min=31707 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32158 max_load_reduction_pct=0.492813 moved_to_replicas=65441 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[30364, 38496, 33985, 45773, 15030, 31504, 27097, 39895] avg=32768.0000 max=45773 max_rank=3 min=15030 min_rank=4 imbalance_ratio=1.396881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[24413, 65254, 42492, 7107, 32762, 56646, 37, 33433] avg=32768.0000 max=65254 max_rank=1 min=37 min_rank=6 imbalance_ratio=1.991394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] avg=32768.0000 max=33096 max_rank=0 min=31707 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32158 max_load_reduction_pct=0.492813 moved_to_replicas=65441 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31504 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12676 max_load_reduction_pct=0.276932 moved_to_replicas=25765 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[30364, 38496, 33985, 45773, 15030, 31504, 27097, 39895] avg=32768.0000 max=45773 max_rank=3 min=15030 min_rank=4 imbalance_ratio=1.396881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31504 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12676 max_load_reduction_pct=0.276932 moved_to_replicas=25765 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[30364, 38496, 33985, 45773, 15030, 31504, 27097, 39895] avg=32768.0000 max=45773 max_rank=3 min=15030 min_rank=4 imbalance_ratio=1.396881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31707, 32762, 33096, 32195, 33096] avg=32768.0000 max=33096 max_rank=0 min=31707 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32158 max_load_reduction_pct=0.492813 moved_to_replicas=65441 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[30364, 38496, 33985, 45773, 15030, 31504, 27097, 39895] avg=32768.0000 max=45773 max_rank=3 min=15030 min_rank=4 imbalance_ratio=1.396881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31504 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12676 max_load_reduction_pct=0.276932 moved_to_replicas=25765 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[30364, 38496, 33985, 45773, 15030, 31504, 27097, 39895] avg=32768.0000 max=45773 max_rank=3 min=15030 min_rank=4 imbalance_ratio=1.396881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31504 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12676 max_load_reduction_pct=0.276932 moved_to_replicas=25765 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[36360, 32672, 6795, 6916, 101829, 53346, 589, 23637] avg=32768.0000 max=101829 max_rank=4 min=589 min_rank=6 imbalance_ratio=3.107574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[30364, 38496, 33985, 45773, 15030, 31504, 27097, 39895] avg=32768.0000 max=45773 max_rank=3 min=15030 min_rank=4 imbalance_ratio=1.396881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31504 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12676 max_load_reduction_pct=0.276932 moved_to_replicas=25765 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[36360, 32672, 6795, 6916, 101829, 53346, 589, 23637] avg=32768.0000 max=101829 max_rank=4 min=589 min_rank=6 imbalance_ratio=3.107574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31504 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12676 max_load_reduction_pct=0.276932 moved_to_replicas=25765 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[30364, 38496, 33985, 45773, 15030, 31504, 27097, 39895] avg=32768.0000 max=45773 max_rank=3 min=15030 min_rank=4 imbalance_ratio=1.396881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] avg=32768.0000 max=33097 max_rank=2 min=30896 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=68732 max_load_reduction_pct=0.674975 moved_to_replicas=92247 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31504 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12676 max_load_reduction_pct=0.276932 moved_to_replicas=25765 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[36360, 32672, 6795, 6916, 101829, 53346, 589, 23637] avg=32768.0000 max=101829 max_rank=4 min=589 min_rank=6 imbalance_ratio=3.107574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] avg=32768.0000 max=33097 max_rank=2 min=30896 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=68732 max_load_reduction_pct=0.674975 moved_to_replicas=92247 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[36360, 32672, 6795, 6916, 101829, 53346, 589, 23637] avg=32768.0000 max=101829 max_rank=4 min=589 min_rank=6 imbalance_ratio=3.107574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32064, 33096, 33096, 33097, 33095, 31504, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31504 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12676 max_load_reduction_pct=0.276932 moved_to_replicas=25765 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[36360, 32672, 6795, 6916, 101829, 53346, 589, 23637] avg=32768.0000 max=101829 max_rank=4 min=589 min_rank=6 imbalance_ratio=3.107574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[36360, 32672, 6795, 6916, 101829, 53346, 589, 23637] avg=32768.0000 max=101829 max_rank=4 min=589 min_rank=6 imbalance_ratio=3.107574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] avg=32768.0000 max=33097 max_rank=2 min=30896 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=68732 max_load_reduction_pct=0.674975 moved_to_replicas=92247 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] avg=32768.0000 max=33097 max_rank=2 min=30896 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=68732 max_load_reduction_pct=0.674975 moved_to_replicas=92247 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[36360, 32672, 6795, 6916, 101829, 53346, 589, 23637] avg=32768.0000 max=101829 max_rank=4 min=589 min_rank=6 imbalance_ratio=3.107574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] avg=32768.0000 max=33097 max_rank=2 min=30896 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=68732 max_load_reduction_pct=0.674975 moved_to_replicas=92247 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] avg=32768.0000 max=33097 max_rank=2 min=30896 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=68732 max_load_reduction_pct=0.674975 moved_to_replicas=92247 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[36360, 32672, 6795, 6916, 101829, 53346, 589, 23637] avg=32768.0000 max=101829 max_rank=4 min=589 min_rank=6 imbalance_ratio=3.107574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] avg=32768.0000 max=33097 max_rank=2 min=30896 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=68732 max_load_reduction_pct=0.674975 moved_to_replicas=92247 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32672, 33097, 33094, 33096, 33097, 33096, 30896] avg=32768.0000 max=33097 max_rank=2 min=30896 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=68732 max_load_reduction_pct=0.674975 moved_to_replicas=92247 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=10 total_expected_recv=330623 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=71936 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=10 total_expected_recv=327558 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74880 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=10 total_expected_recv=323836 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78336 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=10 total_expected_recv=326310 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=10 total_expected_recv=328837 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=73472 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=10 total_expected_recv=324448 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78592 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=10 total_expected_recv=330152 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=72320 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=10 total_expected_recv=329676 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=72448 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:51:59] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 244.7 | throughput per GPU (TFLOP/s/GPU): 504.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.109299E+01 | loss scale: 1.0 | grad norm: 107.031 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33920 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=31096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=31096 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=31037 expected_overflow=0 exact_recv_per_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=31037 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32221 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32221 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31348 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31348 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=31964 expected_overflow=0 exact_recv_per_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=31964 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=31927 expected_overflow=0 exact_recv_per_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=31927 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32411 expected_overflow=0 exact_recv_per_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32411 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=31719 expected_overflow=0 exact_recv_per_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=31719 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=31124 expected_overflow=0 exact_recv_per_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=31124 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=31846 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=31846 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32397 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32397 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=31734 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=31734 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=31726 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=31726 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=31967 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=31967 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[26498, 9701, 29136, 54964, 20075, 61497, 30908, 29365] avg=32768.0000 max=61497 max_rank=5 min=9701 min_rank=1 imbalance_ratio=1.876740 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[26498, 9701, 29136, 54964, 20075, 61497, 30908, 29365] avg=32768.0000 max=61497 max_rank=5 min=9701 min_rank=1 imbalance_ratio=1.876740 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[26498, 9701, 29136, 54964, 20075, 61497, 30908, 29365] avg=32768.0000 max=61497 max_rank=5 min=9701 min_rank=1 imbalance_ratio=1.876740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[26498, 9701, 29136, 54964, 20075, 61497, 30908, 29365] avg=32768.0000 max=61497 max_rank=5 min=9701 min_rank=1 imbalance_ratio=1.876740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] avg=32768.0000 max=33097 max_rank=1 min=31096 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28400 max_load_reduction_pct=0.461811 moved_to_replicas=50269 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[26498, 9701, 29136, 54964, 20075, 61497, 30908, 29365] avg=32768.0000 max=61497 max_rank=5 min=9701 min_rank=1 imbalance_ratio=1.876740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[26498, 9701, 29136, 54964, 20075, 61497, 30908, 29365] avg=32768.0000 max=61497 max_rank=5 min=9701 min_rank=1 imbalance_ratio=1.876740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] avg=32768.0000 max=33097 max_rank=1 min=31096 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28400 max_load_reduction_pct=0.461811 moved_to_replicas=50269 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] avg=32768.0000 max=33097 max_rank=1 min=31096 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28400 max_load_reduction_pct=0.461811 moved_to_replicas=50269 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[26498, 9701, 29136, 54964, 20075, 61497, 30908, 29365] avg=32768.0000 max=61497 max_rank=5 min=9701 min_rank=1 imbalance_ratio=1.876740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] avg=32768.0000 max=33097 max_rank=1 min=31096 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28400 max_load_reduction_pct=0.461811 moved_to_replicas=50269 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[26498, 9701, 29136, 54964, 20075, 61497, 30908, 29365] avg=32768.0000 max=61497 max_rank=5 min=9701 min_rank=1 imbalance_ratio=1.876740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] avg=32768.0000 max=33097 max_rank=1 min=31096 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28400 max_load_reduction_pct=0.461811 moved_to_replicas=50269 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] avg=32768.0000 max=33097 max_rank=1 min=31096 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28400 max_load_reduction_pct=0.461811 moved_to_replicas=50269 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[94638, 5174, 23305, 52421, 1572, 14938, 2879, 67217] avg=32768.0000 max=94638 max_rank=0 min=1572 min_rank=4 imbalance_ratio=2.888123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[94638, 5174, 23305, 52421, 1572, 14938, 2879, 67217] avg=32768.0000 max=94638 max_rank=0 min=1572 min_rank=4 imbalance_ratio=2.888123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] avg=32768.0000 max=33097 max_rank=1 min=31096 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28400 max_load_reduction_pct=0.461811 moved_to_replicas=50269 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33096, 33097, 33095, 33097, 31096, 33096, 32472, 33095] avg=32768.0000 max=33097 max_rank=1 min=31096 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28400 max_load_reduction_pct=0.461811 moved_to_replicas=50269 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] avg=32768.0000 max=33097 max_rank=7 min=31037 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=61541 max_load_reduction_pct=0.650278 moved_to_replicas=114988 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[94638, 5174, 23305, 52421, 1572, 14938, 2879, 67217] avg=32768.0000 max=94638 max_rank=0 min=1572 min_rank=4 imbalance_ratio=2.888123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[94638, 5174, 23305, 52421, 1572, 14938, 2879, 67217] avg=32768.0000 max=94638 max_rank=0 min=1572 min_rank=4 imbalance_ratio=2.888123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[94638, 5174, 23305, 52421, 1572, 14938, 2879, 67217] avg=32768.0000 max=94638 max_rank=0 min=1572 min_rank=4 imbalance_ratio=2.888123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[94638, 5174, 23305, 52421, 1572, 14938, 2879, 67217] avg=32768.0000 max=94638 max_rank=0 min=1572 min_rank=4 imbalance_ratio=2.888123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] avg=32768.0000 max=33097 max_rank=7 min=31037 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=61541 max_load_reduction_pct=0.650278 moved_to_replicas=114988 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] avg=32768.0000 max=33097 max_rank=7 min=31037 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=61541 max_load_reduction_pct=0.650278 moved_to_replicas=114988 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[94638, 5174, 23305, 52421, 1572, 14938, 2879, 67217] avg=32768.0000 max=94638 max_rank=0 min=1572 min_rank=4 imbalance_ratio=2.888123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[94638, 5174, 23305, 52421, 1572, 14938, 2879, 67217] avg=32768.0000 max=94638 max_rank=0 min=1572 min_rank=4 imbalance_ratio=2.888123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[84860, 16257, 29971, 51994, 32221, 40284, 87, 6470] avg=32768.0000 max=84860 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.589722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] avg=32768.0000 max=33097 max_rank=7 min=31037 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=61541 max_load_reduction_pct=0.650278 moved_to_replicas=114988 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] avg=32768.0000 max=33097 max_rank=7 min=31037 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=61541 max_load_reduction_pct=0.650278 moved_to_replicas=114988 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] avg=32768.0000 max=33097 max_rank=7 min=31037 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=61541 max_load_reduction_pct=0.650278 moved_to_replicas=114988 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[84860, 16257, 29971, 51994, 32221, 40284, 87, 6470] avg=32768.0000 max=84860 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.589722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] avg=32768.0000 max=33097 max_rank=7 min=31037 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=61541 max_load_reduction_pct=0.650278 moved_to_replicas=114988 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31037, 32729, 33096, 33096, 33096, 32897, 33097] avg=32768.0000 max=33097 max_rank=7 min=31037 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=61541 max_load_reduction_pct=0.650278 moved_to_replicas=114988 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] avg=32768.0000 max=33097 max_rank=0 min=31348 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=51763 max_load_reduction_pct=0.609981 moved_to_replicas=77850 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[84860, 16257, 29971, 51994, 32221, 40284, 87, 6470] avg=32768.0000 max=84860 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.589722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[84860, 16257, 29971, 51994, 32221, 40284, 87, 6470] avg=32768.0000 max=84860 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.589722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[84860, 16257, 29971, 51994, 32221, 40284, 87, 6470] avg=32768.0000 max=84860 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.589722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] avg=32768.0000 max=33097 max_rank=0 min=31348 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=51763 max_load_reduction_pct=0.609981 moved_to_replicas=77850 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[84860, 16257, 29971, 51994, 32221, 40284, 87, 6470] avg=32768.0000 max=84860 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.589722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] avg=32768.0000 max=33097 max_rank=0 min=31348 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=51763 max_load_reduction_pct=0.609981 moved_to_replicas=77850 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[84860, 16257, 29971, 51994, 32221, 40284, 87, 6470] avg=32768.0000 max=84860 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.589722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] avg=32768.0000 max=33097 max_rank=0 min=31348 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=51763 max_load_reduction_pct=0.609981 moved_to_replicas=77850 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[84860, 16257, 29971, 51994, 32221, 40284, 87, 6470] avg=32768.0000 max=84860 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.589722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[31927, 23291, 17167, 55647, 45300, 34023, 22018, 32771] avg=32768.0000 max=55647 max_rank=3 min=17167 min_rank=2 imbalance_ratio=1.698212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] avg=32768.0000 max=33097 max_rank=0 min=31348 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=51763 max_load_reduction_pct=0.609981 moved_to_replicas=77850 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] avg=32768.0000 max=33097 max_rank=0 min=31348 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=51763 max_load_reduction_pct=0.609981 moved_to_replicas=77850 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[31927, 23291, 17167, 55647, 45300, 34023, 22018, 32771] avg=32768.0000 max=55647 max_rank=3 min=17167 min_rank=2 imbalance_ratio=1.698212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] avg=32768.0000 max=33097 max_rank=0 min=31348 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=51763 max_load_reduction_pct=0.609981 moved_to_replicas=77850 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[31927, 23291, 17167, 55647, 45300, 34023, 22018, 32771] avg=32768.0000 max=55647 max_rank=3 min=17167 min_rank=2 imbalance_ratio=1.698212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[33097, 33094, 33096, 33096, 32221, 33096, 31348, 33096] avg=32768.0000 max=33097 max_rank=0 min=31348 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=51763 max_load_reduction_pct=0.609981 moved_to_replicas=77850 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] avg=32768.0000 max=33097 max_rank=2 min=31927 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=22550 max_load_reduction_pct=0.405233 moved_to_replicas=35682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[31927, 23291, 17167, 55647, 45300, 34023, 22018, 32771] avg=32768.0000 max=55647 max_rank=3 min=17167 min_rank=2 imbalance_ratio=1.698212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] avg=32768.0000 max=33097 max_rank=2 min=31927 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=22550 max_load_reduction_pct=0.405233 moved_to_replicas=35682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[31927, 23291, 17167, 55647, 45300, 34023, 22018, 32771] avg=32768.0000 max=55647 max_rank=3 min=17167 min_rank=2 imbalance_ratio=1.698212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[31927, 23291, 17167, 55647, 45300, 34023, 22018, 32771] avg=32768.0000 max=55647 max_rank=3 min=17167 min_rank=2 imbalance_ratio=1.698212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] avg=32768.0000 max=33097 max_rank=2 min=31927 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=22550 max_load_reduction_pct=0.405233 moved_to_replicas=35682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[31927, 23291, 17167, 55647, 45300, 34023, 22018, 32771] avg=32768.0000 max=55647 max_rank=3 min=17167 min_rank=2 imbalance_ratio=1.698212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] avg=32768.0000 max=33097 max_rank=2 min=31927 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=22550 max_load_reduction_pct=0.405233 moved_to_replicas=35682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[31927, 23291, 17167, 55647, 45300, 34023, 22018, 32771] avg=32768.0000 max=55647 max_rank=3 min=17167 min_rank=2 imbalance_ratio=1.698212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[66863, 27, 98901, 64491, 31679, 107, 42, 34] avg=32768.0000 max=98901 max_rank=2 min=27 min_rank=1 imbalance_ratio=3.018219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] avg=32768.0000 max=33097 max_rank=2 min=31927 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=22550 max_load_reduction_pct=0.405233 moved_to_replicas=35682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] avg=32768.0000 max=33097 max_rank=2 min=31927 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=22550 max_load_reduction_pct=0.405233 moved_to_replicas=35682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[66863, 27, 98901, 64491, 31679, 107, 42, 34] avg=32768.0000 max=98901 max_rank=2 min=27 min_rank=1 imbalance_ratio=3.018219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] avg=32768.0000 max=33097 max_rank=2 min=31927 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=22550 max_load_reduction_pct=0.405233 moved_to_replicas=35682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[31927, 33096, 33097, 33097, 33096, 33096, 31964, 32771] avg=32768.0000 max=33097 max_rank=2 min=31927 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=22550 max_load_reduction_pct=0.405233 moved_to_replicas=35682 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[66863, 27, 98901, 64491, 31679, 107, 42, 34] avg=32768.0000 max=98901 max_rank=2 min=27 min_rank=1 imbalance_ratio=3.018219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] avg=32768.0000 max=33096 max_rank=0 min=31719 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65805 max_load_reduction_pct=0.665362 moved_to_replicas=130967 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[66863, 27, 98901, 64491, 31679, 107, 42, 34] avg=32768.0000 max=98901 max_rank=2 min=27 min_rank=1 imbalance_ratio=3.018219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] avg=32768.0000 max=33096 max_rank=0 min=31719 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65805 max_load_reduction_pct=0.665362 moved_to_replicas=130967 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[66863, 27, 98901, 64491, 31679, 107, 42, 34] avg=32768.0000 max=98901 max_rank=2 min=27 min_rank=1 imbalance_ratio=3.018219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[66863, 27, 98901, 64491, 31679, 107, 42, 34] avg=32768.0000 max=98901 max_rank=2 min=27 min_rank=1 imbalance_ratio=3.018219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] avg=32768.0000 max=33096 max_rank=0 min=31719 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65805 max_load_reduction_pct=0.665362 moved_to_replicas=130967 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] avg=32768.0000 max=33096 max_rank=0 min=31719 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65805 max_load_reduction_pct=0.665362 moved_to_replicas=130967 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[66863, 27, 98901, 64491, 31679, 107, 42, 34] avg=32768.0000 max=98901 max_rank=2 min=27 min_rank=1 imbalance_ratio=3.018219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[66863, 27, 98901, 64491, 31679, 107, 42, 34] avg=32768.0000 max=98901 max_rank=2 min=27 min_rank=1 imbalance_ratio=3.018219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[42, 6425, 34119, 75152, 34982, 32770, 16724, 61930] avg=32768.0000 max=75152 max_rank=3 min=42 min_rank=0 imbalance_ratio=2.293457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] avg=32768.0000 max=33096 max_rank=0 min=31719 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65805 max_load_reduction_pct=0.665362 moved_to_replicas=130967 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] avg=32768.0000 max=33096 max_rank=0 min=31719 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65805 max_load_reduction_pct=0.665362 moved_to_replicas=130967 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[42, 6425, 34119, 75152, 34982, 32770, 16724, 61930] avg=32768.0000 max=75152 max_rank=3 min=42 min_rank=0 imbalance_ratio=2.293457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] avg=32768.0000 max=33096 max_rank=0 min=31719 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65805 max_load_reduction_pct=0.665362 moved_to_replicas=130967 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[42, 6425, 34119, 75152, 34982, 32770, 16724, 61930] avg=32768.0000 max=75152 max_rank=3 min=42 min_rank=0 imbalance_ratio=2.293457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32780, 33096, 33096, 32411, 32850, 33096, 31719] avg=32768.0000 max=33096 max_rank=0 min=31719 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65805 max_load_reduction_pct=0.665362 moved_to_replicas=130967 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] avg=32768.0000 max=33097 max_rank=1 min=31124 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42055 max_load_reduction_pct=0.559599 moved_to_replicas=73799 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[42, 6425, 34119, 75152, 34982, 32770, 16724, 61930] avg=32768.0000 max=75152 max_rank=3 min=42 min_rank=0 imbalance_ratio=2.293457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] avg=32768.0000 max=33097 max_rank=1 min=31124 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42055 max_load_reduction_pct=0.559599 moved_to_replicas=73799 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[42, 6425, 34119, 75152, 34982, 32770, 16724, 61930] avg=32768.0000 max=75152 max_rank=3 min=42 min_rank=0 imbalance_ratio=2.293457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[42, 6425, 34119, 75152, 34982, 32770, 16724, 61930] avg=32768.0000 max=75152 max_rank=3 min=42 min_rank=0 imbalance_ratio=2.293457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] avg=32768.0000 max=33097 max_rank=1 min=31124 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42055 max_load_reduction_pct=0.559599 moved_to_replicas=73799 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[42, 6425, 34119, 75152, 34982, 32770, 16724, 61930] avg=32768.0000 max=75152 max_rank=3 min=42 min_rank=0 imbalance_ratio=2.293457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] avg=32768.0000 max=33097 max_rank=1 min=31124 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42055 max_load_reduction_pct=0.559599 moved_to_replicas=73799 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] avg=32768.0000 max=33097 max_rank=1 min=31124 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42055 max_load_reduction_pct=0.559599 moved_to_replicas=73799 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] avg=32768.0000 max=33097 max_rank=1 min=31124 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42055 max_load_reduction_pct=0.559599 moved_to_replicas=73799 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[42, 6425, 34119, 75152, 34982, 32770, 16724, 61930] avg=32768.0000 max=75152 max_rank=3 min=42 min_rank=0 imbalance_ratio=2.293457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65070, 32766, 32728, 28870, 5, 490, 70369, 31846] avg=32768.0000 max=70369 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.147491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65070, 32766, 32728, 28870, 5, 490, 70369, 31846] avg=32768.0000 max=70369 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.147491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] avg=32768.0000 max=33097 max_rank=1 min=31124 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42055 max_load_reduction_pct=0.559599 moved_to_replicas=73799 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65070, 32766, 32728, 28870, 5, 490, 70369, 31846] avg=32768.0000 max=70369 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.147491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32770, 33097, 33096, 33095, 33096, 32770, 31124, 33096] avg=32768.0000 max=33097 max_rank=1 min=31124 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42055 max_load_reduction_pct=0.559599 moved_to_replicas=73799 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65070, 32766, 32728, 28870, 5, 490, 70369, 31846] avg=32768.0000 max=70369 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.147491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] avg=32768.0000 max=33096 max_rank=0 min=31846 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37273 max_load_reduction_pct=0.529679 moved_to_replicas=69247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65070, 32766, 32728, 28870, 5, 490, 70369, 31846] avg=32768.0000 max=70369 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.147491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] avg=32768.0000 max=33096 max_rank=0 min=31846 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37273 max_load_reduction_pct=0.529679 moved_to_replicas=69247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65070, 32766, 32728, 28870, 5, 490, 70369, 31846] avg=32768.0000 max=70369 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.147491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] avg=32768.0000 max=33096 max_rank=0 min=31846 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37273 max_load_reduction_pct=0.529679 moved_to_replicas=69247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65070, 32766, 32728, 28870, 5, 490, 70369, 31846] avg=32768.0000 max=70369 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.147491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] avg=32768.0000 max=33096 max_rank=0 min=31846 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37273 max_load_reduction_pct=0.529679 moved_to_replicas=69247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] avg=32768.0000 max=33096 max_rank=0 min=31846 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37273 max_load_reduction_pct=0.529679 moved_to_replicas=69247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65070, 32766, 32728, 28870, 5, 490, 70369, 31846] avg=32768.0000 max=70369 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.147491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] avg=32768.0000 max=33096 max_rank=0 min=31846 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37273 max_load_reduction_pct=0.529679 moved_to_replicas=69247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[65482, 38262, 32848, 11, 32780, 38153, 9837, 44771] avg=32768.0000 max=65482 max_rank=0 min=11 min_rank=3 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[65482, 38262, 32848, 11, 32780, 38153, 9837, 44771] avg=32768.0000 max=65482 max_rank=0 min=11 min_rank=3 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] avg=32768.0000 max=33096 max_rank=0 min=31846 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37273 max_load_reduction_pct=0.529679 moved_to_replicas=69247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[65482, 38262, 32848, 11, 32780, 38153, 9837, 44771] avg=32768.0000 max=65482 max_rank=0 min=11 min_rank=3 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[65482, 38262, 32848, 11, 32780, 38153, 9837, 44771] avg=32768.0000 max=65482 max_rank=0 min=11 min_rank=3 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32766, 32728, 32764, 32752, 33096, 33096, 31846] avg=32768.0000 max=33096 max_rank=0 min=31846 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37273 max_load_reduction_pct=0.529679 moved_to_replicas=69247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] avg=32768.0000 max=33098 max_rank=1 min=31734 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32384 max_load_reduction_pct=0.494548 moved_to_replicas=54284 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] avg=32768.0000 max=33098 max_rank=1 min=31734 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32384 max_load_reduction_pct=0.494548 moved_to_replicas=54284 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[65482, 38262, 32848, 11, 32780, 38153, 9837, 44771] avg=32768.0000 max=65482 max_rank=0 min=11 min_rank=3 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[65482, 38262, 32848, 11, 32780, 38153, 9837, 44771] avg=32768.0000 max=65482 max_rank=0 min=11 min_rank=3 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] avg=32768.0000 max=33098 max_rank=1 min=31734 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32384 max_load_reduction_pct=0.494548 moved_to_replicas=54284 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[65482, 38262, 32848, 11, 32780, 38153, 9837, 44771] avg=32768.0000 max=65482 max_rank=0 min=11 min_rank=3 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] avg=32768.0000 max=33098 max_rank=1 min=31734 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32384 max_load_reduction_pct=0.494548 moved_to_replicas=54284 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] avg=32768.0000 max=33098 max_rank=1 min=31734 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32384 max_load_reduction_pct=0.494548 moved_to_replicas=54284 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32772, 1544, 32763, 35176, 13, 75, 29269, 130532] avg=32768.0000 max=130532 max_rank=7 min=13 min_rank=4 imbalance_ratio=3.983521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[65482, 38262, 32848, 11, 32780, 38153, 9837, 44771] avg=32768.0000 max=65482 max_rank=0 min=11 min_rank=3 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32772, 1544, 32763, 35176, 13, 75, 29269, 130532] avg=32768.0000 max=130532 max_rank=7 min=13 min_rank=4 imbalance_ratio=3.983521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] avg=32768.0000 max=33098 max_rank=1 min=31734 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32384 max_load_reduction_pct=0.494548 moved_to_replicas=54284 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] avg=32768.0000 max=33098 max_rank=1 min=31734 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32384 max_load_reduction_pct=0.494548 moved_to_replicas=54284 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32772, 1544, 32763, 35176, 13, 75, 29269, 130532] avg=32768.0000 max=130532 max_rank=7 min=13 min_rank=4 imbalance_ratio=3.983521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32772, 1544, 32763, 35176, 13, 75, 29269, 130532] avg=32768.0000 max=130532 max_rank=7 min=13 min_rank=4 imbalance_ratio=3.983521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] avg=32768.0000 max=33096 max_rank=1 min=31726 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97436 max_load_reduction_pct=0.746453 moved_to_replicas=99516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33098, 32848, 32397, 32780, 33096, 31734, 33095] avg=32768.0000 max=33098 max_rank=1 min=31734 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32384 max_load_reduction_pct=0.494548 moved_to_replicas=54284 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] avg=32768.0000 max=33096 max_rank=1 min=31726 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97436 max_load_reduction_pct=0.746453 moved_to_replicas=99516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32772, 1544, 32763, 35176, 13, 75, 29269, 130532] avg=32768.0000 max=130532 max_rank=7 min=13 min_rank=4 imbalance_ratio=3.983521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] avg=32768.0000 max=33096 max_rank=1 min=31726 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97436 max_load_reduction_pct=0.746453 moved_to_replicas=99516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32772, 1544, 32763, 35176, 13, 75, 29269, 130532] avg=32768.0000 max=130532 max_rank=7 min=13 min_rank=4 imbalance_ratio=3.983521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32772, 1544, 32763, 35176, 13, 75, 29269, 130532] avg=32768.0000 max=130532 max_rank=7 min=13 min_rank=4 imbalance_ratio=3.983521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] avg=32768.0000 max=33096 max_rank=1 min=31726 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97436 max_load_reduction_pct=0.746453 moved_to_replicas=99516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] avg=32768.0000 max=33096 max_rank=1 min=31726 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97436 max_load_reduction_pct=0.746453 moved_to_replicas=99516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65697, 7, 32772, 32763, 23274, 64769, 10117, 32745] avg=32768.0000 max=65697 max_rank=0 min=7 min_rank=1 imbalance_ratio=2.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32772, 1544, 32763, 35176, 13, 75, 29269, 130532] avg=32768.0000 max=130532 max_rank=7 min=13 min_rank=4 imbalance_ratio=3.983521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65697, 7, 32772, 32763, 23274, 64769, 10117, 32745] avg=32768.0000 max=65697 max_rank=0 min=7 min_rank=1 imbalance_ratio=2.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] avg=32768.0000 max=33096 max_rank=1 min=31726 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97436 max_load_reduction_pct=0.746453 moved_to_replicas=99516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] avg=32768.0000 max=33096 max_rank=1 min=31726 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97436 max_load_reduction_pct=0.746453 moved_to_replicas=99516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65697, 7, 32772, 32763, 23274, 64769, 10117, 32745] avg=32768.0000 max=65697 max_rank=0 min=7 min_rank=1 imbalance_ratio=2.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] avg=32768.0000 max=33097 max_rank=6 min=31967 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32600 max_load_reduction_pct=0.496217 moved_to_replicas=64274 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65697, 7, 32772, 32763, 23274, 64769, 10117, 32745] avg=32768.0000 max=65697 max_rank=0 min=7 min_rank=1 imbalance_ratio=2.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65697, 7, 32772, 32763, 23274, 64769, 10117, 32745] avg=32768.0000 max=65697 max_rank=0 min=7 min_rank=1 imbalance_ratio=2.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] avg=32768.0000 max=33097 max_rank=6 min=31967 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32600 max_load_reduction_pct=0.496217 moved_to_replicas=64274 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32772, 33096, 32763, 33096, 32774, 32821, 31726, 33096] avg=32768.0000 max=33096 max_rank=1 min=31726 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97436 max_load_reduction_pct=0.746453 moved_to_replicas=99516 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] avg=32768.0000 max=33097 max_rank=6 min=31967 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32600 max_load_reduction_pct=0.496217 moved_to_replicas=64274 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65697, 7, 32772, 32763, 23274, 64769, 10117, 32745] avg=32768.0000 max=65697 max_rank=0 min=7 min_rank=1 imbalance_ratio=2.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] avg=32768.0000 max=33097 max_rank=6 min=31967 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32600 max_load_reduction_pct=0.496217 moved_to_replicas=64274 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65697, 7, 32772, 32763, 23274, 64769, 10117, 32745] avg=32768.0000 max=65697 max_rank=0 min=7 min_rank=1 imbalance_ratio=2.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] avg=32768.0000 max=33097 max_rank=6 min=31967 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32600 max_load_reduction_pct=0.496217 moved_to_replicas=64274 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65697, 7, 32772, 32763, 23274, 64769, 10117, 32745] avg=32768.0000 max=65697 max_rank=0 min=7 min_rank=1 imbalance_ratio=2.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] avg=32768.0000 max=33097 max_rank=6 min=31967 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32600 max_load_reduction_pct=0.496217 moved_to_replicas=64274 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] avg=32768.0000 max=33097 max_rank=6 min=31967 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32600 max_load_reduction_pct=0.496217 moved_to_replicas=64274 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32608, 32772, 32763, 31967, 33096, 33097, 32745] avg=32768.0000 max=33097 max_rank=6 min=31967 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32600 max_load_reduction_pct=0.496217 moved_to_replicas=64274 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=10 total_expected_recv=325289 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=10 total_expected_recv=329597 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74368 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=10 total_expected_recv=329320 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74752 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=10 total_expected_recv=330113 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=73728 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=10 total_expected_recv=327656 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75008 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=10 total_expected_recv=329142 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74496 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=10 total_expected_recv=322554 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79616 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=10 total_expected_recv=327769 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75392 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:52:00] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 241.8 | throughput per GPU (TFLOP/s/GPU): 510.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.000580E+01 | loss scale: 1.0 | grad norm: 26.604 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=30837 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=30837 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=31139 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=31139 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=31213 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=31213 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32355 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32355 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32322 expected_overflow=0 exact_recv_per_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32322 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=31841 expected_overflow=0 exact_recv_per_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=31841 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=31142 expected_overflow=0 exact_recv_per_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=31142 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32202 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=31965 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32202 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=31965 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32293 expected_overflow=0 exact_recv_per_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=31276 expected_overflow=0 exact_recv_per_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32293 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=31276 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32167 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32167 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=31820 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=31820 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32406 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32406 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[25366, 15228, 29492, 52616, 19164, 59457, 28090, 32731] avg=32768.0000 max=59457 max_rank=5 min=15228 min_rank=1 imbalance_ratio=1.814484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[25366, 15228, 29492, 52616, 19164, 59457, 28090, 32731] avg=32768.0000 max=59457 max_rank=5 min=15228 min_rank=1 imbalance_ratio=1.814484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[25366, 15228, 29492, 52616, 19164, 59457, 28090, 32731] avg=32768.0000 max=59457 max_rank=5 min=15228 min_rank=1 imbalance_ratio=1.814484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=0 min=30837 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26361 max_load_reduction_pct=0.443362 moved_to_replicas=45881 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=0 min=30837 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26361 max_load_reduction_pct=0.443362 moved_to_replicas=45881 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=0 min=30837 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26361 max_load_reduction_pct=0.443362 moved_to_replicas=45881 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[25366, 15228, 29492, 52616, 19164, 59457, 28090, 32731] avg=32768.0000 max=59457 max_rank=5 min=15228 min_rank=1 imbalance_ratio=1.814484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[25366, 15228, 29492, 52616, 19164, 59457, 28090, 32731] avg=32768.0000 max=59457 max_rank=5 min=15228 min_rank=1 imbalance_ratio=1.814484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[25366, 15228, 29492, 52616, 19164, 59457, 28090, 32731] avg=32768.0000 max=59457 max_rank=5 min=15228 min_rank=1 imbalance_ratio=1.814484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[25366, 15228, 29492, 52616, 19164, 59457, 28090, 32731] avg=32768.0000 max=59457 max_rank=5 min=15228 min_rank=1 imbalance_ratio=1.814484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[25366, 15228, 29492, 52616, 19164, 59457, 28090, 32731] avg=32768.0000 max=59457 max_rank=5 min=15228 min_rank=1 imbalance_ratio=1.814484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[64420, 2064, 102484, 35009, 338, 1586, 45870, 10373] avg=32768.0000 max=102484 max_rank=2 min=338 min_rank=4 imbalance_ratio=3.127563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[64420, 2064, 102484, 35009, 338, 1586, 45870, 10373] avg=32768.0000 max=102484 max_rank=2 min=338 min_rank=4 imbalance_ratio=3.127563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[64420, 2064, 102484, 35009, 338, 1586, 45870, 10373] avg=32768.0000 max=102484 max_rank=2 min=338 min_rank=4 imbalance_ratio=3.127563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=0 min=30837 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26361 max_load_reduction_pct=0.443362 moved_to_replicas=45881 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=0 min=30837 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26361 max_load_reduction_pct=0.443362 moved_to_replicas=45881 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=0 min=30837 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26361 max_load_reduction_pct=0.443362 moved_to_replicas=45881 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=0 min=30837 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26361 max_load_reduction_pct=0.443362 moved_to_replicas=45881 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[33096, 33096, 30837, 33096, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=0 min=30837 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26361 max_load_reduction_pct=0.443362 moved_to_replicas=45881 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31139 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69387 max_load_reduction_pct=0.677052 moved_to_replicas=115399 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31139 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69387 max_load_reduction_pct=0.677052 moved_to_replicas=115399 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31139 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69387 max_load_reduction_pct=0.677052 moved_to_replicas=115399 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[64420, 2064, 102484, 35009, 338, 1586, 45870, 10373] avg=32768.0000 max=102484 max_rank=2 min=338 min_rank=4 imbalance_ratio=3.127563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[64420, 2064, 102484, 35009, 338, 1586, 45870, 10373] avg=32768.0000 max=102484 max_rank=2 min=338 min_rank=4 imbalance_ratio=3.127563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[64420, 2064, 102484, 35009, 338, 1586, 45870, 10373] avg=32768.0000 max=102484 max_rank=2 min=338 min_rank=4 imbalance_ratio=3.127563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[64420, 2064, 102484, 35009, 338, 1586, 45870, 10373] avg=32768.0000 max=102484 max_rank=2 min=338 min_rank=4 imbalance_ratio=3.127563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[64420, 2064, 102484, 35009, 338, 1586, 45870, 10373] avg=32768.0000 max=102484 max_rank=2 min=338 min_rank=4 imbalance_ratio=3.127563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[1470, 79147, 32355, 5776, 21376, 59324, 48143, 14553] avg=32768.0000 max=79147 max_rank=1 min=1470 min_rank=0 imbalance_ratio=2.415375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[1470, 79147, 32355, 5776, 21376, 59324, 48143, 14553] avg=32768.0000 max=79147 max_rank=1 min=1470 min_rank=0 imbalance_ratio=2.415375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[1470, 79147, 32355, 5776, 21376, 59324, 48143, 14553] avg=32768.0000 max=79147 max_rank=1 min=1470 min_rank=0 imbalance_ratio=2.415375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31139 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69387 max_load_reduction_pct=0.677052 moved_to_replicas=115399 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31139 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69387 max_load_reduction_pct=0.677052 moved_to_replicas=115399 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31139 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69387 max_load_reduction_pct=0.677052 moved_to_replicas=115399 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31139 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69387 max_load_reduction_pct=0.677052 moved_to_replicas=115399 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 31139, 32429, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31139 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69387 max_load_reduction_pct=0.677052 moved_to_replicas=115399 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=31213 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46050 max_load_reduction_pct=0.581829 moved_to_replicas=87326 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=31213 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46050 max_load_reduction_pct=0.581829 moved_to_replicas=87326 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=31213 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46050 max_load_reduction_pct=0.581829 moved_to_replicas=87326 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[1470, 79147, 32355, 5776, 21376, 59324, 48143, 14553] avg=32768.0000 max=79147 max_rank=1 min=1470 min_rank=0 imbalance_ratio=2.415375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[1470, 79147, 32355, 5776, 21376, 59324, 48143, 14553] avg=32768.0000 max=79147 max_rank=1 min=1470 min_rank=0 imbalance_ratio=2.415375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[1470, 79147, 32355, 5776, 21376, 59324, 48143, 14553] avg=32768.0000 max=79147 max_rank=1 min=1470 min_rank=0 imbalance_ratio=2.415375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[1470, 79147, 32355, 5776, 21376, 59324, 48143, 14553] avg=32768.0000 max=79147 max_rank=1 min=1470 min_rank=0 imbalance_ratio=2.415375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[1470, 79147, 32355, 5776, 21376, 59324, 48143, 14553] avg=32768.0000 max=79147 max_rank=1 min=1470 min_rank=0 imbalance_ratio=2.415375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[25, 116, 31952, 94237, 32769, 33466, 37044, 32535] avg=32768.0000 max=94237 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.875885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[25, 116, 31952, 94237, 32769, 33466, 37044, 32535] avg=32768.0000 max=94237 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.875885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[25, 116, 31952, 94237, 32769, 33466, 37044, 32535] avg=32768.0000 max=94237 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.875885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=31213 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46050 max_load_reduction_pct=0.581829 moved_to_replicas=87326 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=31213 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46050 max_load_reduction_pct=0.581829 moved_to_replicas=87326 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=31213 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46050 max_load_reduction_pct=0.581829 moved_to_replicas=87326 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=31213 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46050 max_load_reduction_pct=0.581829 moved_to_replicas=87326 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33096, 32355, 31213, 33097, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=31213 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46050 max_load_reduction_pct=0.581829 moved_to_replicas=87326 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] avg=32768.0000 max=33096 max_rank=3 min=32322 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61141 max_load_reduction_pct=0.648800 moved_to_replicas=65459 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] avg=32768.0000 max=33096 max_rank=3 min=32322 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61141 max_load_reduction_pct=0.648800 moved_to_replicas=65459 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] avg=32768.0000 max=33096 max_rank=3 min=32322 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61141 max_load_reduction_pct=0.648800 moved_to_replicas=65459 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[25, 116, 31952, 94237, 32769, 33466, 37044, 32535] avg=32768.0000 max=94237 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.875885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[25, 116, 31952, 94237, 32769, 33466, 37044, 32535] avg=32768.0000 max=94237 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.875885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[25, 116, 31952, 94237, 32769, 33466, 37044, 32535] avg=32768.0000 max=94237 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.875885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[25, 116, 31952, 94237, 32769, 33466, 37044, 32535] avg=32768.0000 max=94237 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.875885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[25, 116, 31952, 94237, 32769, 33466, 37044, 32535] avg=32768.0000 max=94237 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.875885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65572, 32513, 326, 97405, 33337, 32943, 6, 42] avg=32768.0000 max=97405 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.972565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65572, 32513, 326, 97405, 33337, 32943, 6, 42] avg=32768.0000 max=97405 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.972565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65572, 32513, 326, 97405, 33337, 32943, 6, 42] avg=32768.0000 max=97405 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.972565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] avg=32768.0000 max=33096 max_rank=3 min=32322 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61141 max_load_reduction_pct=0.648800 moved_to_replicas=65459 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] avg=32768.0000 max=33096 max_rank=3 min=32322 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61141 max_load_reduction_pct=0.648800 moved_to_replicas=65459 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] avg=32768.0000 max=33096 max_rank=3 min=32322 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61141 max_load_reduction_pct=0.648800 moved_to_replicas=65459 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] avg=32768.0000 max=33096 max_rank=3 min=32322 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61141 max_load_reduction_pct=0.648800 moved_to_replicas=65459 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32783, 32447, 32322, 33096, 32769, 33096, 33096, 32535] avg=32768.0000 max=33096 max_rank=3 min=32322 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61141 max_load_reduction_pct=0.648800 moved_to_replicas=65459 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] avg=32768.0000 max=33096 max_rank=0 min=31841 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64309 max_load_reduction_pct=0.660223 moved_to_replicas=97026 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] avg=32768.0000 max=33096 max_rank=0 min=31841 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64309 max_load_reduction_pct=0.660223 moved_to_replicas=97026 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] avg=32768.0000 max=33096 max_rank=0 min=31841 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64309 max_load_reduction_pct=0.660223 moved_to_replicas=97026 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65572, 32513, 326, 97405, 33337, 32943, 6, 42] avg=32768.0000 max=97405 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.972565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65572, 32513, 326, 97405, 33337, 32943, 6, 42] avg=32768.0000 max=97405 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.972565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65572, 32513, 326, 97405, 33337, 32943, 6, 42] avg=32768.0000 max=97405 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.972565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65572, 32513, 326, 97405, 33337, 32943, 6, 42] avg=32768.0000 max=97405 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.972565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65572, 32513, 326, 97405, 33337, 32943, 6, 42] avg=32768.0000 max=97405 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.972565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32757, 19988, 137, 64248, 35922, 5, 15056, 94031] avg=32768.0000 max=94031 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.869598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32757, 19988, 137, 64248, 35922, 5, 15056, 94031] avg=32768.0000 max=94031 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.869598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32757, 19988, 137, 64248, 35922, 5, 15056, 94031] avg=32768.0000 max=94031 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.869598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] avg=32768.0000 max=33096 max_rank=0 min=31841 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64309 max_load_reduction_pct=0.660223 moved_to_replicas=97026 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] avg=32768.0000 max=33096 max_rank=0 min=31841 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64309 max_load_reduction_pct=0.660223 moved_to_replicas=97026 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] avg=32768.0000 max=33096 max_rank=0 min=31841 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64309 max_load_reduction_pct=0.660223 moved_to_replicas=97026 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] avg=32768.0000 max=33096 max_rank=0 min=31841 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64309 max_load_reduction_pct=0.660223 moved_to_replicas=97026 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] avg=32768.0000 max=33097 max_rank=1 min=31142 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60934 max_load_reduction_pct=0.648020 moved_to_replicas=94913 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32513, 32802, 33096, 33096, 32943, 32757, 31841] avg=32768.0000 max=33096 max_rank=0 min=31841 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64309 max_load_reduction_pct=0.660223 moved_to_replicas=97026 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] avg=32768.0000 max=33097 max_rank=1 min=31142 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60934 max_load_reduction_pct=0.648020 moved_to_replicas=94913 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] avg=32768.0000 max=33097 max_rank=1 min=31142 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60934 max_load_reduction_pct=0.648020 moved_to_replicas=94913 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32757, 19988, 137, 64248, 35922, 5, 15056, 94031] avg=32768.0000 max=94031 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.869598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32757, 19988, 137, 64248, 35922, 5, 15056, 94031] avg=32768.0000 max=94031 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.869598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32757, 19988, 137, 64248, 35922, 5, 15056, 94031] avg=32768.0000 max=94031 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.869598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32757, 19988, 137, 64248, 35922, 5, 15056, 94031] avg=32768.0000 max=94031 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.869598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[64985, 10, 31965, 32808, 1645, 62662, 35284, 32785] avg=32768.0000 max=64985 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.983185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32757, 19988, 137, 64248, 35922, 5, 15056, 94031] avg=32768.0000 max=94031 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.869598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[64985, 10, 31965, 32808, 1645, 62662, 35284, 32785] avg=32768.0000 max=64985 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.983185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[64985, 10, 31965, 32808, 1645, 62662, 35284, 32785] avg=32768.0000 max=64985 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.983185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] avg=32768.0000 max=33097 max_rank=1 min=31142 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60934 max_load_reduction_pct=0.648020 moved_to_replicas=94913 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] avg=32768.0000 max=33097 max_rank=1 min=31142 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60934 max_load_reduction_pct=0.648020 moved_to_replicas=94913 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] avg=32768.0000 max=33097 max_rank=1 min=31142 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60934 max_load_reduction_pct=0.648020 moved_to_replicas=94913 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] avg=32768.0000 max=33097 max_rank=1 min=31142 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60934 max_load_reduction_pct=0.648020 moved_to_replicas=94913 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=31889 max_load_reduction_pct=0.490713 moved_to_replicas=63643 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32757, 33097, 31142, 33096, 33096, 32766, 33095, 33095] avg=32768.0000 max=33097 max_rank=1 min=31142 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60934 max_load_reduction_pct=0.648020 moved_to_replicas=94913 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=31889 max_load_reduction_pct=0.490713 moved_to_replicas=63643 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=31889 max_load_reduction_pct=0.490713 moved_to_replicas=63643 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[64985, 10, 31965, 32808, 1645, 62662, 35284, 32785] avg=32768.0000 max=64985 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.983185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[64985, 10, 31965, 32808, 1645, 62662, 35284, 32785] avg=32768.0000 max=64985 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.983185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[64985, 10, 31965, 32808, 1645, 62662, 35284, 32785] avg=32768.0000 max=64985 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.983185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[64985, 10, 31965, 32808, 1645, 62662, 35284, 32785] avg=32768.0000 max=64985 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.983185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[48938, 58981, 55487, 4, 33185, 65385, 56, 108] avg=32768.0000 max=65385 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.995392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[64985, 10, 31965, 32808, 1645, 62662, 35284, 32785] avg=32768.0000 max=64985 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.983185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[48938, 58981, 55487, 4, 33185, 65385, 56, 108] avg=32768.0000 max=65385 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.995392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[48938, 58981, 55487, 4, 33185, 65385, 56, 108] avg=32768.0000 max=65385 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.995392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=31889 max_load_reduction_pct=0.490713 moved_to_replicas=63643 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=31889 max_load_reduction_pct=0.490713 moved_to_replicas=63643 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=31889 max_load_reduction_pct=0.490713 moved_to_replicas=63643 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=31889 max_load_reduction_pct=0.490713 moved_to_replicas=63643 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] avg=32768.0000 max=33098 max_rank=7 min=31276 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32287 max_load_reduction_pct=0.493798 moved_to_replicas=96496 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32202, 31965, 32808, 33096, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=31889 max_load_reduction_pct=0.490713 moved_to_replicas=63643 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] avg=32768.0000 max=33098 max_rank=7 min=31276 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32287 max_load_reduction_pct=0.493798 moved_to_replicas=96496 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] avg=32768.0000 max=33098 max_rank=7 min=31276 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32287 max_load_reduction_pct=0.493798 moved_to_replicas=96496 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[48938, 58981, 55487, 4, 33185, 65385, 56, 108] avg=32768.0000 max=65385 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.995392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[48938, 58981, 55487, 4, 33185, 65385, 56, 108] avg=32768.0000 max=65385 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.995392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[48938, 58981, 55487, 4, 33185, 65385, 56, 108] avg=32768.0000 max=65385 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.995392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[48938, 58981, 55487, 4, 33185, 65385, 56, 108] avg=32768.0000 max=65385 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.995392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[3, 65474, 33497, 24423, 40439, 65524, 13, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[3, 65474, 33497, 24423, 40439, 65524, 13, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[48938, 58981, 55487, 4, 33185, 65385, 56, 108] avg=32768.0000 max=65385 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.995392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[3, 65474, 33497, 24423, 40439, 65524, 13, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] avg=32768.0000 max=33098 max_rank=7 min=31276 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32287 max_load_reduction_pct=0.493798 moved_to_replicas=96496 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] avg=32768.0000 max=33098 max_rank=7 min=31276 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32287 max_load_reduction_pct=0.493798 moved_to_replicas=96496 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] avg=32768.0000 max=33098 max_rank=7 min=31276 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32287 max_load_reduction_pct=0.493798 moved_to_replicas=96496 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] avg=32768.0000 max=33098 max_rank=7 min=31276 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32287 max_load_reduction_pct=0.493798 moved_to_replicas=96496 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] avg=32768.0000 max=33096 max_rank=1 min=32167 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=72550 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] avg=32768.0000 max=33096 max_rank=1 min=32167 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=72550 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33095, 33096, 32293, 33096, 33096, 31276, 33098] avg=32768.0000 max=33098 max_rank=7 min=31276 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32287 max_load_reduction_pct=0.493798 moved_to_replicas=96496 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] avg=32768.0000 max=33096 max_rank=1 min=32167 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=72550 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[3, 65474, 33497, 24423, 40439, 65524, 13, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[3, 65474, 33497, 24423, 40439, 65524, 13, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[3, 65474, 33497, 24423, 40439, 65524, 13, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[3, 65474, 33497, 24423, 40439, 65524, 13, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65533, 5, 65497, 3, 56443, 58411, 16235, 17] avg=32768.0000 max=65533 max_rank=0 min=3 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65533, 5, 65497, 3, 56443, 58411, 16235, 17] avg=32768.0000 max=65533 max_rank=0 min=3 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[3, 65474, 33497, 24423, 40439, 65524, 13, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65533, 5, 65497, 3, 56443, 58411, 16235, 17] avg=32768.0000 max=65533 max_rank=0 min=3 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] avg=32768.0000 max=33096 max_rank=1 min=32167 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=72550 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] avg=32768.0000 max=33096 max_rank=1 min=32167 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=72550 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] avg=32768.0000 max=33096 max_rank=1 min=32167 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=72550 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] avg=32768.0000 max=33096 max_rank=1 min=32167 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=72550 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] avg=32768.0000 max=33096 max_rank=0 min=31820 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=113500 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] avg=32768.0000 max=33096 max_rank=0 min=31820 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=113500 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] avg=32768.0000 max=33096 max_rank=0 min=31820 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=113500 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 33096, 32167, 33096, 33096, 32391, 32771] avg=32768.0000 max=33096 max_rank=1 min=32167 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=72550 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65533, 5, 65497, 3, 56443, 58411, 16235, 17] avg=32768.0000 max=65533 max_rank=0 min=3 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65533, 5, 65497, 3, 56443, 58411, 16235, 17] avg=32768.0000 max=65533 max_rank=0 min=3 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65533, 5, 65497, 3, 56443, 58411, 16235, 17] avg=32768.0000 max=65533 max_rank=0 min=3 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65533, 5, 65497, 3, 56443, 58411, 16235, 17] avg=32768.0000 max=65533 max_rank=0 min=3 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] avg=32768.0000 max=33096 max_rank=0 min=31820 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=113500 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65533, 5, 65497, 3, 56443, 58411, 16235, 17] avg=32768.0000 max=65533 max_rank=0 min=3 min_rank=3 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] avg=32768.0000 max=33096 max_rank=0 min=31820 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=113500 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] avg=32768.0000 max=33096 max_rank=0 min=31820 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=113500 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] avg=32768.0000 max=33096 max_rank=0 min=31820 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=113500 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32406, 33096, 32440, 33095, 33096, 33095, 31820] avg=32768.0000 max=33096 max_rank=0 min=31820 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=113500 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=10 total_expected_recv=329809 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75136 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=10 total_expected_recv=326870 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=10 total_expected_recv=328676 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75648 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=10 total_expected_recv=323807 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=80000 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=10 total_expected_recv=329641 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74752 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=10 total_expected_recv=326401 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=10 total_expected_recv=328143 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75520 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=10 total_expected_recv=328093 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75776 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:52:00] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 236.7 | throughput per GPU (TFLOP/s/GPU): 521.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.050145E+01 | loss scale: 1.0 | grad norm: 8.744 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=31097 expected_overflow=0 exact_recv_per_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=31097 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32601 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32601 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32202 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32202 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32338 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32338 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33099 expected_overflow=0 exact_recv_per_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33099 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=31369 expected_overflow=0 exact_recv_per_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=31369 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=31811 expected_overflow=0 exact_recv_per_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=31811 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32079 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32079 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32483 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32483 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=31563 expected_overflow=0 exact_recv_per_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=31563 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33099 expected_overflow=0 exact_recv_per_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33099 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=31479 expected_overflow=0 exact_recv_per_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=31479 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32402 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32402 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[15906, 36217, 37284, 32471, 14566, 44618, 40845, 40237] avg=32768.0000 max=44618 max_rank=5 min=14566 min_rank=4 imbalance_ratio=1.361633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[15906, 36217, 37284, 32471, 14566, 44618, 40845, 40237] avg=32768.0000 max=44618 max_rank=5 min=14566 min_rank=4 imbalance_ratio=1.361633 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[15906, 36217, 37284, 32471, 14566, 44618, 40845, 40237] avg=32768.0000 max=44618 max_rank=5 min=14566 min_rank=4 imbalance_ratio=1.361633 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[15906, 36217, 37284, 32471, 14566, 44618, 40845, 40237] avg=32768.0000 max=44618 max_rank=5 min=14566 min_rank=4 imbalance_ratio=1.361633 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31097 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=11521 max_load_reduction_pct=0.258214 moved_to_replicas=33721 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31097 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=11521 max_load_reduction_pct=0.258214 moved_to_replicas=33721 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[15906, 36217, 37284, 32471, 14566, 44618, 40845, 40237] avg=32768.0000 max=44618 max_rank=5 min=14566 min_rank=4 imbalance_ratio=1.361633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31097 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=11521 max_load_reduction_pct=0.258214 moved_to_replicas=33721 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31097 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=11521 max_load_reduction_pct=0.258214 moved_to_replicas=33721 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[15906, 36217, 37284, 32471, 14566, 44618, 40845, 40237] avg=32768.0000 max=44618 max_rank=5 min=14566 min_rank=4 imbalance_ratio=1.361633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[15906, 36217, 37284, 32471, 14566, 44618, 40845, 40237] avg=32768.0000 max=44618 max_rank=5 min=14566 min_rank=4 imbalance_ratio=1.361633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[15906, 36217, 37284, 32471, 14566, 44618, 40845, 40237] avg=32768.0000 max=44618 max_rank=5 min=14566 min_rank=4 imbalance_ratio=1.361633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31097 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=11521 max_load_reduction_pct=0.258214 moved_to_replicas=33721 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[765, 27597, 32792, 32580, 32601, 78375, 16061, 41373] avg=32768.0000 max=78375 max_rank=5 min=765 min_rank=0 imbalance_ratio=2.391815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31097 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=11521 max_load_reduction_pct=0.258214 moved_to_replicas=33721 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[765, 27597, 32792, 32580, 32601, 78375, 16061, 41373] avg=32768.0000 max=78375 max_rank=5 min=765 min_rank=0 imbalance_ratio=2.391815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[765, 27597, 32792, 32580, 32601, 78375, 16061, 41373] avg=32768.0000 max=78375 max_rank=5 min=765 min_rank=0 imbalance_ratio=2.391815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[765, 27597, 32792, 32580, 32601, 78375, 16061, 41373] avg=32768.0000 max=78375 max_rank=5 min=765 min_rank=0 imbalance_ratio=2.391815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31097 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=11521 max_load_reduction_pct=0.258214 moved_to_replicas=33721 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[31097, 33096, 33095, 32471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31097 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=11521 max_load_reduction_pct=0.258214 moved_to_replicas=33721 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] avg=32768.0000 max=33097 max_rank=1 min=31787 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=45278 max_load_reduction_pct=0.577710 moved_to_replicas=53556 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] avg=32768.0000 max=33097 max_rank=1 min=31787 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=45278 max_load_reduction_pct=0.577710 moved_to_replicas=53556 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[765, 27597, 32792, 32580, 32601, 78375, 16061, 41373] avg=32768.0000 max=78375 max_rank=5 min=765 min_rank=0 imbalance_ratio=2.391815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] avg=32768.0000 max=33097 max_rank=1 min=31787 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=45278 max_load_reduction_pct=0.577710 moved_to_replicas=53556 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] avg=32768.0000 max=33097 max_rank=1 min=31787 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=45278 max_load_reduction_pct=0.577710 moved_to_replicas=53556 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[765, 27597, 32792, 32580, 32601, 78375, 16061, 41373] avg=32768.0000 max=78375 max_rank=5 min=765 min_rank=0 imbalance_ratio=2.391815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] avg=32768.0000 max=33097 max_rank=1 min=31787 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=45278 max_load_reduction_pct=0.577710 moved_to_replicas=53556 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[765, 27597, 32792, 32580, 32601, 78375, 16061, 41373] avg=32768.0000 max=78375 max_rank=5 min=765 min_rank=0 imbalance_ratio=2.391815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[765, 27597, 32792, 32580, 32601, 78375, 16061, 41373] avg=32768.0000 max=78375 max_rank=5 min=765 min_rank=0 imbalance_ratio=2.391815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[38694, 65429, 32750, 32795, 16731, 32769, 5, 42971] avg=32768.0000 max=65429 max_rank=1 min=5 min_rank=6 imbalance_ratio=1.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] avg=32768.0000 max=33097 max_rank=1 min=31787 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=45278 max_load_reduction_pct=0.577710 moved_to_replicas=53556 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[38694, 65429, 32750, 32795, 16731, 32769, 5, 42971] avg=32768.0000 max=65429 max_rank=1 min=5 min_rank=6 imbalance_ratio=1.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[38694, 65429, 32750, 32795, 16731, 32769, 5, 42971] avg=32768.0000 max=65429 max_rank=1 min=5 min_rank=6 imbalance_ratio=1.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[38694, 65429, 32750, 32795, 16731, 32769, 5, 42971] avg=32768.0000 max=65429 max_rank=1 min=5 min_rank=6 imbalance_ratio=1.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] avg=32768.0000 max=33097 max_rank=1 min=31787 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=45278 max_load_reduction_pct=0.577710 moved_to_replicas=53556 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33097, 32792, 32580, 32601, 33096, 31787, 33095] avg=32768.0000 max=33097 max_rank=1 min=31787 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=45278 max_load_reduction_pct=0.577710 moved_to_replicas=53556 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] avg=32768.0000 max=33098 max_rank=0 min=32202 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32331 max_load_reduction_pct=0.494139 moved_to_replicas=47806 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[38694, 65429, 32750, 32795, 16731, 32769, 5, 42971] avg=32768.0000 max=65429 max_rank=1 min=5 min_rank=6 imbalance_ratio=1.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] avg=32768.0000 max=33098 max_rank=0 min=32202 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32331 max_load_reduction_pct=0.494139 moved_to_replicas=47806 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] avg=32768.0000 max=33098 max_rank=0 min=32202 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32331 max_load_reduction_pct=0.494139 moved_to_replicas=47806 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] avg=32768.0000 max=33098 max_rank=0 min=32202 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32331 max_load_reduction_pct=0.494139 moved_to_replicas=47806 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[38694, 65429, 32750, 32795, 16731, 32769, 5, 42971] avg=32768.0000 max=65429 max_rank=1 min=5 min_rank=6 imbalance_ratio=1.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] avg=32768.0000 max=33098 max_rank=0 min=32202 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32331 max_load_reduction_pct=0.494139 moved_to_replicas=47806 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[38694, 65429, 32750, 32795, 16731, 32769, 5, 42971] avg=32768.0000 max=65429 max_rank=1 min=5 min_rank=6 imbalance_ratio=1.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[28848, 32761, 27461, 32774, 61538, 42924, 35618, 220] avg=32768.0000 max=61538 max_rank=4 min=220 min_rank=7 imbalance_ratio=1.877991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[38694, 65429, 32750, 32795, 16731, 32769, 5, 42971] avg=32768.0000 max=65429 max_rank=1 min=5 min_rank=6 imbalance_ratio=1.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] avg=32768.0000 max=33098 max_rank=0 min=32202 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32331 max_load_reduction_pct=0.494139 moved_to_replicas=47806 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[28848, 32761, 27461, 32774, 61538, 42924, 35618, 220] avg=32768.0000 max=61538 max_rank=4 min=220 min_rank=7 imbalance_ratio=1.877991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[28848, 32761, 27461, 32774, 61538, 42924, 35618, 220] avg=32768.0000 max=61538 max_rank=4 min=220 min_rank=7 imbalance_ratio=1.877991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[28848, 32761, 27461, 32774, 61538, 42924, 35618, 220] avg=32768.0000 max=61538 max_rank=4 min=220 min_rank=7 imbalance_ratio=1.877991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] avg=32768.0000 max=33098 max_rank=0 min=32202 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32331 max_load_reduction_pct=0.494139 moved_to_replicas=47806 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] avg=32768.0000 max=33099 max_rank=5 min=31369 min_rank=0 imbalance_ratio=1.010101 max_load_reduction=28439 max_load_reduction_pct=0.462137 moved_to_replicas=40792 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[33098, 33096, 32750, 32795, 32202, 32769, 32338, 33096] avg=32768.0000 max=33098 max_rank=0 min=32202 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32331 max_load_reduction_pct=0.494139 moved_to_replicas=47806 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[28848, 32761, 27461, 32774, 61538, 42924, 35618, 220] avg=32768.0000 max=61538 max_rank=4 min=220 min_rank=7 imbalance_ratio=1.877991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] avg=32768.0000 max=33099 max_rank=5 min=31369 min_rank=0 imbalance_ratio=1.010101 max_load_reduction=28439 max_load_reduction_pct=0.462137 moved_to_replicas=40792 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] avg=32768.0000 max=33099 max_rank=5 min=31369 min_rank=0 imbalance_ratio=1.010101 max_load_reduction=28439 max_load_reduction_pct=0.462137 moved_to_replicas=40792 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] avg=32768.0000 max=33099 max_rank=5 min=31369 min_rank=0 imbalance_ratio=1.010101 max_load_reduction=28439 max_load_reduction_pct=0.462137 moved_to_replicas=40792 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[28848, 32761, 27461, 32774, 61538, 42924, 35618, 220] avg=32768.0000 max=61538 max_rank=4 min=220 min_rank=7 imbalance_ratio=1.877991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] avg=32768.0000 max=33099 max_rank=5 min=31369 min_rank=0 imbalance_ratio=1.010101 max_load_reduction=28439 max_load_reduction_pct=0.462137 moved_to_replicas=40792 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32789, 29, 63591, 23635, 64878, 2477, 41883, 32862] avg=32768.0000 max=64878 max_rank=4 min=29 min_rank=1 imbalance_ratio=1.979919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[28848, 32761, 27461, 32774, 61538, 42924, 35618, 220] avg=32768.0000 max=61538 max_rank=4 min=220 min_rank=7 imbalance_ratio=1.877991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] avg=32768.0000 max=33099 max_rank=5 min=31369 min_rank=0 imbalance_ratio=1.010101 max_load_reduction=28439 max_load_reduction_pct=0.462137 moved_to_replicas=40792 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[28848, 32761, 27461, 32774, 61538, 42924, 35618, 220] avg=32768.0000 max=61538 max_rank=4 min=220 min_rank=7 imbalance_ratio=1.877991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32789, 29, 63591, 23635, 64878, 2477, 41883, 32862] avg=32768.0000 max=64878 max_rank=4 min=29 min_rank=1 imbalance_ratio=1.979919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32789, 29, 63591, 23635, 64878, 2477, 41883, 32862] avg=32768.0000 max=64878 max_rank=4 min=29 min_rank=1 imbalance_ratio=1.979919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32789, 29, 63591, 23635, 64878, 2477, 41883, 32862] avg=32768.0000 max=64878 max_rank=4 min=29 min_rank=1 imbalance_ratio=1.979919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] avg=32768.0000 max=33097 max_rank=6 min=31811 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31781 max_load_reduction_pct=0.489858 moved_to_replicas=71064 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] avg=32768.0000 max=33099 max_rank=5 min=31369 min_rank=0 imbalance_ratio=1.010101 max_load_reduction=28439 max_load_reduction_pct=0.462137 moved_to_replicas=40792 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32789, 29, 63591, 23635, 64878, 2477, 41883, 32862] avg=32768.0000 max=64878 max_rank=4 min=29 min_rank=1 imbalance_ratio=1.979919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[31369, 32761, 33095, 32774, 33095, 33099, 33097, 32854] avg=32768.0000 max=33099 max_rank=5 min=31369 min_rank=0 imbalance_ratio=1.010101 max_load_reduction=28439 max_load_reduction_pct=0.462137 moved_to_replicas=40792 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] avg=32768.0000 max=33097 max_rank=6 min=31811 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31781 max_load_reduction_pct=0.489858 moved_to_replicas=71064 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] avg=32768.0000 max=33097 max_rank=6 min=31811 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31781 max_load_reduction_pct=0.489858 moved_to_replicas=71064 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] avg=32768.0000 max=33097 max_rank=6 min=31811 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31781 max_load_reduction_pct=0.489858 moved_to_replicas=71064 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32789, 29, 63591, 23635, 64878, 2477, 41883, 32862] avg=32768.0000 max=64878 max_rank=4 min=29 min_rank=1 imbalance_ratio=1.979919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] avg=32768.0000 max=33097 max_rank=6 min=31811 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31781 max_load_reduction_pct=0.489858 moved_to_replicas=71064 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[7, 97939, 32879, 1, 65737, 2, 65536, 43] avg=32768.0000 max=97939 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.988861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] avg=32768.0000 max=33097 max_rank=6 min=31811 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31781 max_load_reduction_pct=0.489858 moved_to_replicas=71064 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32789, 29, 63591, 23635, 64878, 2477, 41883, 32862] avg=32768.0000 max=64878 max_rank=4 min=29 min_rank=1 imbalance_ratio=1.979919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[7, 97939, 32879, 1, 65737, 2, 65536, 43] avg=32768.0000 max=97939 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.988861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32789, 29, 63591, 23635, 64878, 2477, 41883, 32862] avg=32768.0000 max=64878 max_rank=4 min=29 min_rank=1 imbalance_ratio=1.979919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[7, 97939, 32879, 1, 65737, 2, 65536, 43] avg=32768.0000 max=97939 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.988861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[7, 97939, 32879, 1, 65737, 2, 65536, 43] avg=32768.0000 max=97939 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.988861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] avg=32768.0000 max=33096 max_rank=1 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64843 max_load_reduction_pct=0.662075 moved_to_replicas=129924 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[7, 97939, 32879, 1, 65737, 2, 65536, 43] avg=32768.0000 max=97939 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.988861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] avg=32768.0000 max=33097 max_rank=6 min=31811 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31781 max_load_reduction_pct=0.489858 moved_to_replicas=71064 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] avg=32768.0000 max=33096 max_rank=1 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64843 max_load_reduction_pct=0.662075 moved_to_replicas=129924 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32789, 31811, 33096, 32421, 33096, 32972, 33097, 32862] avg=32768.0000 max=33097 max_rank=6 min=31811 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31781 max_load_reduction_pct=0.489858 moved_to_replicas=71064 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[7, 97939, 32879, 1, 65737, 2, 65536, 43] avg=32768.0000 max=97939 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.988861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] avg=32768.0000 max=33096 max_rank=1 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64843 max_load_reduction_pct=0.662075 moved_to_replicas=129924 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] avg=32768.0000 max=33096 max_rank=1 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64843 max_load_reduction_pct=0.662075 moved_to_replicas=129924 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] avg=32768.0000 max=33096 max_rank=1 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64843 max_load_reduction_pct=0.662075 moved_to_replicas=129924 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[21764, 79400, 32583, 33102, 47451, 15000, 32836, 8] avg=32768.0000 max=79400 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.423096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] avg=32768.0000 max=33096 max_rank=1 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64843 max_load_reduction_pct=0.662075 moved_to_replicas=129924 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[21764, 79400, 32583, 33102, 47451, 15000, 32836, 8] avg=32768.0000 max=79400 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.423096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[7, 97939, 32879, 1, 65737, 2, 65536, 43] avg=32768.0000 max=97939 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.988861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[7, 97939, 32879, 1, 65737, 2, 65536, 43] avg=32768.0000 max=97939 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.988861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[21764, 79400, 32583, 33102, 47451, 15000, 32836, 8] avg=32768.0000 max=79400 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.423096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[21764, 79400, 32583, 33102, 47451, 15000, 32836, 8] avg=32768.0000 max=79400 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.423096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] avg=32768.0000 max=33099 max_rank=1 min=31563 min_rank=5 imbalance_ratio=1.010101 max_load_reduction=46301 max_load_reduction_pct=0.583136 moved_to_replicas=60665 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[21764, 79400, 32583, 33102, 47451, 15000, 32836, 8] avg=32768.0000 max=79400 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.423096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] avg=32768.0000 max=33099 max_rank=1 min=31563 min_rank=5 imbalance_ratio=1.010101 max_load_reduction=46301 max_load_reduction_pct=0.583136 moved_to_replicas=60665 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] avg=32768.0000 max=33096 max_rank=1 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64843 max_load_reduction_pct=0.662075 moved_to_replicas=129924 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32648, 33096, 32879, 32767, 33096, 32079, 33096, 32483] avg=32768.0000 max=33096 max_rank=1 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64843 max_load_reduction_pct=0.662075 moved_to_replicas=129924 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] avg=32768.0000 max=33099 max_rank=1 min=31563 min_rank=5 imbalance_ratio=1.010101 max_load_reduction=46301 max_load_reduction_pct=0.583136 moved_to_replicas=60665 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[21764, 79400, 32583, 33102, 47451, 15000, 32836, 8] avg=32768.0000 max=79400 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.423096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] avg=32768.0000 max=33099 max_rank=1 min=31563 min_rank=5 imbalance_ratio=1.010101 max_load_reduction=46301 max_load_reduction_pct=0.583136 moved_to_replicas=60665 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] avg=32768.0000 max=33099 max_rank=1 min=31563 min_rank=5 imbalance_ratio=1.010101 max_load_reduction=46301 max_load_reduction_pct=0.583136 moved_to_replicas=60665 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[65431, 65598, 65467, 0, 22, 23, 32748, 32855] avg=32768.0000 max=65598 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] avg=32768.0000 max=33099 max_rank=1 min=31563 min_rank=5 imbalance_ratio=1.010101 max_load_reduction=46301 max_load_reduction_pct=0.583136 moved_to_replicas=60665 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[65431, 65598, 65467, 0, 22, 23, 32748, 32855] avg=32768.0000 max=65598 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[21764, 79400, 32583, 33102, 47451, 15000, 32836, 8] avg=32768.0000 max=79400 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.423096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[21764, 79400, 32583, 33102, 47451, 15000, 32836, 8] avg=32768.0000 max=79400 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.423096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[65431, 65598, 65467, 0, 22, 23, 32748, 32855] avg=32768.0000 max=65598 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[65431, 65598, 65467, 0, 22, 23, 32748, 32855] avg=32768.0000 max=65598 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] avg=32768.0000 max=33096 max_rank=0 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32502 max_load_reduction_pct=0.495472 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[65431, 65598, 65467, 0, 22, 23, 32748, 32855] avg=32768.0000 max=65598 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] avg=32768.0000 max=33096 max_rank=0 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32502 max_load_reduction_pct=0.495472 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] avg=32768.0000 max=33099 max_rank=1 min=31563 min_rank=5 imbalance_ratio=1.010101 max_load_reduction=46301 max_load_reduction_pct=0.583136 moved_to_replicas=60665 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[33095, 33099, 32583, 33096, 33097, 31563, 32836, 32775] avg=32768.0000 max=33099 max_rank=1 min=31563 min_rank=5 imbalance_ratio=1.010101 max_load_reduction=46301 max_load_reduction_pct=0.583136 moved_to_replicas=60665 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[65431, 65598, 65467, 0, 22, 23, 32748, 32855] avg=32768.0000 max=65598 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] avg=32768.0000 max=33096 max_rank=0 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32502 max_load_reduction_pct=0.495472 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] avg=32768.0000 max=33096 max_rank=0 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32502 max_load_reduction_pct=0.495472 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] avg=32768.0000 max=33096 max_rank=0 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32502 max_load_reduction_pct=0.495472 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32782, 32732, 62111, 1134, 67933, 65432] avg=32768.0000 max=67933 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.073151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32782, 32732, 62111, 1134, 67933, 65432] avg=32768.0000 max=67933 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.073151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] avg=32768.0000 max=33096 max_rank=0 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32502 max_load_reduction_pct=0.495472 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[65431, 65598, 65467, 0, 22, 23, 32748, 32855] avg=32768.0000 max=65598 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[65431, 65598, 65467, 0, 22, 23, 32748, 32855] avg=32768.0000 max=65598 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31479 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34837 max_load_reduction_pct=0.512814 moved_to_replicas=96188 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32782, 32732, 62111, 1134, 67933, 65432] avg=32768.0000 max=67933 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.073151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32782, 32732, 62111, 1134, 67933, 65432] avg=32768.0000 max=67933 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.073151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32782, 32732, 62111, 1134, 67933, 65432] avg=32768.0000 max=67933 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.073151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31479 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34837 max_load_reduction_pct=0.512814 moved_to_replicas=96188 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] avg=32768.0000 max=33096 max_rank=0 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32502 max_load_reduction_pct=0.495472 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32782, 32732, 62111, 1134, 67933, 65432] avg=32768.0000 max=67933 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.073151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32502, 32393, 32358, 32748, 32855] avg=32768.0000 max=33096 max_rank=0 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32502 max_load_reduction_pct=0.495472 moved_to_replicas=97208 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31479 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34837 max_load_reduction_pct=0.512814 moved_to_replicas=96188 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31479 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34837 max_load_reduction_pct=0.512814 moved_to_replicas=96188 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31479 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34837 max_load_reduction_pct=0.512814 moved_to_replicas=96188 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 2, 32767, 65561, 65496, 32781, 0] avg=32768.0000 max=65561 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 2, 32767, 65561, 65496, 32781, 0] avg=32768.0000 max=65561 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31479 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34837 max_load_reduction_pct=0.512814 moved_to_replicas=96188 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32782, 32732, 62111, 1134, 67933, 65432] avg=32768.0000 max=67933 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.073151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] avg=32768.0000 max=33096 max_rank=4 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32465 max_load_reduction_pct=0.495188 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 2, 32767, 65561, 65496, 32781, 0] avg=32768.0000 max=65561 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32782, 32732, 62111, 1134, 67933, 65432] avg=32768.0000 max=67933 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.073151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 2, 32767, 65561, 65496, 32781, 0] avg=32768.0000 max=65561 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 2, 32767, 65561, 65496, 32781, 0] avg=32768.0000 max=65561 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] avg=32768.0000 max=33096 max_rank=4 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32465 max_load_reduction_pct=0.495188 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 2, 32767, 65561, 65496, 32781, 0] avg=32768.0000 max=65561 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31479 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34837 max_load_reduction_pct=0.512814 moved_to_replicas=96188 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] avg=32768.0000 max=33096 max_rank=4 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32465 max_load_reduction_pct=0.495188 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[31479, 32767, 32782, 32732, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31479 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34837 max_load_reduction_pct=0.512814 moved_to_replicas=96188 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] avg=32768.0000 max=33096 max_rank=4 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32465 max_load_reduction_pct=0.495188 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] avg=32768.0000 max=33096 max_rank=4 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32465 max_load_reduction_pct=0.495188 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] avg=32768.0000 max=33096 max_rank=4 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32465 max_load_reduction_pct=0.495188 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 2, 32767, 65561, 65496, 32781, 0] avg=32768.0000 max=65561 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 2, 32767, 65561, 65496, 32781, 0] avg=32768.0000 max=65561 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] avg=32768.0000 max=33096 max_rank=4 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32465 max_load_reduction_pct=0.495188 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32769, 32402, 32767, 33096, 33096, 32781, 32465] avg=32768.0000 max=33096 max_rank=4 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32465 max_load_reduction_pct=0.495188 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=10 total_expected_recv=328869 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=10 total_expected_recv=327224 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78976 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=10 total_expected_recv=326905 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79872 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=10 total_expected_recv=328677 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77568 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=10 total_expected_recv=328570 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=10 total_expected_recv=324535 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=81664 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=10 total_expected_recv=328688 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=10 total_expected_recv=327972 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78848 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:52:00] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 230.1 | throughput per GPU (TFLOP/s/GPU): 536.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.447910E+00 | loss scale: 1.0 | grad norm: 5.497 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32121 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32121 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=31828 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=31828 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32521 expected_overflow=0 exact_recv_per_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32521 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=31638 expected_overflow=0 exact_recv_per_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=31638 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=31477 expected_overflow=0 exact_recv_per_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=31477 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32120 expected_overflow=0 exact_recv_per_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32120 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32118 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=31944 expected_overflow=0 exact_recv_per_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32118 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=31944 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32108 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=31989 expected_overflow=0 exact_recv_per_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32108 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=31989 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32138 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32138 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32301 expected_overflow=0 exact_recv_per_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32301 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[41604, 47976, 39719, 24950, 16329, 27159, 26194, 38213] avg=32768.0000 max=47976 max_rank=1 min=16329 min_rank=4 imbalance_ratio=1.464111 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[41604, 47976, 39719, 24950, 16329, 27159, 26194, 38213] avg=32768.0000 max=47976 max_rank=1 min=16329 min_rank=4 imbalance_ratio=1.464111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14879 max_load_reduction_pct=0.310134 moved_to_replicas=35128 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[41604, 47976, 39719, 24950, 16329, 27159, 26194, 38213] avg=32768.0000 max=47976 max_rank=1 min=16329 min_rank=4 imbalance_ratio=1.464111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[41604, 47976, 39719, 24950, 16329, 27159, 26194, 38213] avg=32768.0000 max=47976 max_rank=1 min=16329 min_rank=4 imbalance_ratio=1.464111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[41604, 47976, 39719, 24950, 16329, 27159, 26194, 38213] avg=32768.0000 max=47976 max_rank=1 min=16329 min_rank=4 imbalance_ratio=1.464111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14879 max_load_reduction_pct=0.310134 moved_to_replicas=35128 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[41604, 47976, 39719, 24950, 16329, 27159, 26194, 38213] avg=32768.0000 max=47976 max_rank=1 min=16329 min_rank=4 imbalance_ratio=1.464111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[41604, 47976, 39719, 24950, 16329, 27159, 26194, 38213] avg=32768.0000 max=47976 max_rank=1 min=16329 min_rank=4 imbalance_ratio=1.464111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[41604, 47976, 39719, 24950, 16329, 27159, 26194, 38213] avg=32768.0000 max=47976 max_rank=1 min=16329 min_rank=4 imbalance_ratio=1.464111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65212, 18664, 50676, 32920, 5, 32891, 27, 61749] avg=32768.0000 max=65212 max_rank=0 min=5 min_rank=4 imbalance_ratio=1.990112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14879 max_load_reduction_pct=0.310134 moved_to_replicas=35128 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14879 max_load_reduction_pct=0.310134 moved_to_replicas=35128 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14879 max_load_reduction_pct=0.310134 moved_to_replicas=35128 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14879 max_load_reduction_pct=0.310134 moved_to_replicas=35128 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14879 max_load_reduction_pct=0.310134 moved_to_replicas=35128 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 30472, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14879 max_load_reduction_pct=0.310134 moved_to_replicas=35128 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] avg=32768.0000 max=33097 max_rank=2 min=31828 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32115 max_load_reduction_pct=0.492471 moved_to_replicas=78349 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65212, 18664, 50676, 32920, 5, 32891, 27, 61749] avg=32768.0000 max=65212 max_rank=0 min=5 min_rank=4 imbalance_ratio=1.990112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65212, 18664, 50676, 32920, 5, 32891, 27, 61749] avg=32768.0000 max=65212 max_rank=0 min=5 min_rank=4 imbalance_ratio=1.990112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65212, 18664, 50676, 32920, 5, 32891, 27, 61749] avg=32768.0000 max=65212 max_rank=0 min=5 min_rank=4 imbalance_ratio=1.990112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65212, 18664, 50676, 32920, 5, 32891, 27, 61749] avg=32768.0000 max=65212 max_rank=0 min=5 min_rank=4 imbalance_ratio=1.990112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] avg=32768.0000 max=33097 max_rank=2 min=31828 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32115 max_load_reduction_pct=0.492471 moved_to_replicas=78349 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65212, 18664, 50676, 32920, 5, 32891, 27, 61749] avg=32768.0000 max=65212 max_rank=0 min=5 min_rank=4 imbalance_ratio=1.990112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65212, 18664, 50676, 32920, 5, 32891, 27, 61749] avg=32768.0000 max=65212 max_rank=0 min=5 min_rank=4 imbalance_ratio=1.990112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] avg=32768.0000 max=33097 max_rank=2 min=31828 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32115 max_load_reduction_pct=0.492471 moved_to_replicas=78349 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[98284, 32835, 98, 67613, 43668, 17503, 1, 2142] avg=32768.0000 max=98284 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65212, 18664, 50676, 32920, 5, 32891, 27, 61749] avg=32768.0000 max=65212 max_rank=0 min=5 min_rank=4 imbalance_ratio=1.990112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] avg=32768.0000 max=33097 max_rank=2 min=31828 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32115 max_load_reduction_pct=0.492471 moved_to_replicas=78349 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] avg=32768.0000 max=33097 max_rank=2 min=31828 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32115 max_load_reduction_pct=0.492471 moved_to_replicas=78349 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] avg=32768.0000 max=33097 max_rank=2 min=31828 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32115 max_load_reduction_pct=0.492471 moved_to_replicas=78349 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] avg=32768.0000 max=33097 max_rank=2 min=31828 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32115 max_load_reduction_pct=0.492471 moved_to_replicas=78349 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=31638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=110277 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33095, 33097, 32920, 32121, 32891, 31828, 33096] avg=32768.0000 max=33097 max_rank=2 min=31828 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32115 max_load_reduction_pct=0.492471 moved_to_replicas=78349 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[98284, 32835, 98, 67613, 43668, 17503, 1, 2142] avg=32768.0000 max=98284 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[98284, 32835, 98, 67613, 43668, 17503, 1, 2142] avg=32768.0000 max=98284 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[98284, 32835, 98, 67613, 43668, 17503, 1, 2142] avg=32768.0000 max=98284 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[98284, 32835, 98, 67613, 43668, 17503, 1, 2142] avg=32768.0000 max=98284 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=31638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=110277 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[98284, 32835, 98, 67613, 43668, 17503, 1, 2142] avg=32768.0000 max=98284 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[98284, 32835, 98, 67613, 43668, 17503, 1, 2142] avg=32768.0000 max=98284 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[48515, 32659, 33593, 38013, 32764, 10643, 244, 65713] avg=32768.0000 max=65713 max_rank=7 min=244 min_rank=6 imbalance_ratio=2.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=31638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=110277 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[98284, 32835, 98, 67613, 43668, 17503, 1, 2142] avg=32768.0000 max=98284 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=31638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=110277 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=31638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=110277 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=31638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=110277 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] avg=32768.0000 max=33096 max_rank=2 min=31477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32617 max_load_reduction_pct=0.496355 moved_to_replicas=53450 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=31638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=110277 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[48515, 32659, 33593, 38013, 32764, 10643, 244, 65713] avg=32768.0000 max=65713 max_rank=7 min=244 min_rank=6 imbalance_ratio=2.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32835, 32521, 33096, 33096, 31638, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=31638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=110277 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[48515, 32659, 33593, 38013, 32764, 10643, 244, 65713] avg=32768.0000 max=65713 max_rank=7 min=244 min_rank=6 imbalance_ratio=2.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[48515, 32659, 33593, 38013, 32764, 10643, 244, 65713] avg=32768.0000 max=65713 max_rank=7 min=244 min_rank=6 imbalance_ratio=2.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[48515, 32659, 33593, 38013, 32764, 10643, 244, 65713] avg=32768.0000 max=65713 max_rank=7 min=244 min_rank=6 imbalance_ratio=2.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] avg=32768.0000 max=33096 max_rank=2 min=31477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32617 max_load_reduction_pct=0.496355 moved_to_replicas=53450 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[48515, 32659, 33593, 38013, 32764, 10643, 244, 65713] avg=32768.0000 max=65713 max_rank=7 min=244 min_rank=6 imbalance_ratio=2.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 29708, 11, 65527, 32775, 358, 68581] avg=32768.0000 max=68581 max_rank=7 min=11 min_rank=3 imbalance_ratio=2.092926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[48515, 32659, 33593, 38013, 32764, 10643, 244, 65713] avg=32768.0000 max=65713 max_rank=7 min=244 min_rank=6 imbalance_ratio=2.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] avg=32768.0000 max=33096 max_rank=2 min=31477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32617 max_load_reduction_pct=0.496355 moved_to_replicas=53450 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] avg=32768.0000 max=33096 max_rank=2 min=31477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32617 max_load_reduction_pct=0.496355 moved_to_replicas=53450 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[48515, 32659, 33593, 38013, 32764, 10643, 244, 65713] avg=32768.0000 max=65713 max_rank=7 min=244 min_rank=6 imbalance_ratio=2.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] avg=32768.0000 max=33096 max_rank=2 min=31477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32617 max_load_reduction_pct=0.496355 moved_to_replicas=53450 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] avg=32768.0000 max=33096 max_rank=2 min=31477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32617 max_load_reduction_pct=0.496355 moved_to_replicas=53450 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35485 max_load_reduction_pct=0.517417 moved_to_replicas=67916 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] avg=32768.0000 max=33096 max_rank=2 min=31477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32617 max_load_reduction_pct=0.496355 moved_to_replicas=53450 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 29708, 11, 65527, 32775, 358, 68581] avg=32768.0000 max=68581 max_rank=7 min=11 min_rank=3 imbalance_ratio=2.092926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[33095, 32659, 33096, 33096, 32764, 31477, 32861, 33096] avg=32768.0000 max=33096 max_rank=2 min=31477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32617 max_load_reduction_pct=0.496355 moved_to_replicas=53450 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 29708, 11, 65527, 32775, 358, 68581] avg=32768.0000 max=68581 max_rank=7 min=11 min_rank=3 imbalance_ratio=2.092926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 29708, 11, 65527, 32775, 358, 68581] avg=32768.0000 max=68581 max_rank=7 min=11 min_rank=3 imbalance_ratio=2.092926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 29708, 11, 65527, 32775, 358, 68581] avg=32768.0000 max=68581 max_rank=7 min=11 min_rank=3 imbalance_ratio=2.092926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35485 max_load_reduction_pct=0.517417 moved_to_replicas=67916 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65536, 149, 32768, 10, 32791, 0, 93047, 37843] avg=32768.0000 max=93047 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.839569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 29708, 11, 65527, 32775, 358, 68581] avg=32768.0000 max=68581 max_rank=7 min=11 min_rank=3 imbalance_ratio=2.092926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 29708, 11, 65527, 32775, 358, 68581] avg=32768.0000 max=68581 max_rank=7 min=11 min_rank=3 imbalance_ratio=2.092926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35485 max_load_reduction_pct=0.517417 moved_to_replicas=67916 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35485 max_load_reduction_pct=0.517417 moved_to_replicas=67916 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 29708, 11, 65527, 32775, 358, 68581] avg=32768.0000 max=68581 max_rank=7 min=11 min_rank=3 imbalance_ratio=2.092926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35485 max_load_reduction_pct=0.517417 moved_to_replicas=67916 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=59951 max_load_reduction_pct=0.644309 moved_to_replicas=97138 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35485 max_load_reduction_pct=0.517417 moved_to_replicas=67916 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35485 max_load_reduction_pct=0.517417 moved_to_replicas=67916 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65536, 149, 32768, 10, 32791, 0, 93047, 37843] avg=32768.0000 max=93047 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.839569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32419, 32765, 32120, 32777, 33096, 32775, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35485 max_load_reduction_pct=0.517417 moved_to_replicas=67916 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65536, 149, 32768, 10, 32791, 0, 93047, 37843] avg=32768.0000 max=93047 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.839569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65536, 149, 32768, 10, 32791, 0, 93047, 37843] avg=32768.0000 max=93047 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.839569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65536, 149, 32768, 10, 32791, 0, 93047, 37843] avg=32768.0000 max=93047 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.839569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[0, 65190, 0, 56833, 3, 10803, 129301, 14] avg=32768.0000 max=129301 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.945953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=59951 max_load_reduction_pct=0.644309 moved_to_replicas=97138 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65536, 149, 32768, 10, 32791, 0, 93047, 37843] avg=32768.0000 max=93047 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.839569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65536, 149, 32768, 10, 32791, 0, 93047, 37843] avg=32768.0000 max=93047 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.839569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=59951 max_load_reduction_pct=0.644309 moved_to_replicas=97138 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=59951 max_load_reduction_pct=0.644309 moved_to_replicas=97138 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=59951 max_load_reduction_pct=0.644309 moved_to_replicas=97138 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65536, 149, 32768, 10, 32791, 0, 93047, 37843] avg=32768.0000 max=93047 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.839569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] avg=32768.0000 max=33097 max_rank=3 min=32108 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=96204 max_load_reduction_pct=0.744031 moved_to_replicas=152036 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=59951 max_load_reduction_pct=0.644309 moved_to_replicas=97138 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=59951 max_load_reduction_pct=0.644309 moved_to_replicas=97138 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[0, 65190, 0, 56833, 3, 10803, 129301, 14] avg=32768.0000 max=129301 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.945953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32589, 32768, 31944, 32791, 32767, 33096, 33093] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=59951 max_load_reduction_pct=0.644309 moved_to_replicas=97138 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[0, 65190, 0, 56833, 3, 10803, 129301, 14] avg=32768.0000 max=129301 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.945953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[0, 65190, 0, 56833, 3, 10803, 129301, 14] avg=32768.0000 max=129301 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.945953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[0, 65190, 0, 56833, 3, 10803, 129301, 14] avg=32768.0000 max=129301 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.945953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32766, 214, 65463, 60171, 1, 32768, 0, 70761] avg=32768.0000 max=70761 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.159454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] avg=32768.0000 max=33097 max_rank=3 min=32108 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=96204 max_load_reduction_pct=0.744031 moved_to_replicas=152036 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] avg=32768.0000 max=33097 max_rank=3 min=32108 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=96204 max_load_reduction_pct=0.744031 moved_to_replicas=152036 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[0, 65190, 0, 56833, 3, 10803, 129301, 14] avg=32768.0000 max=129301 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.945953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] avg=32768.0000 max=33097 max_rank=3 min=32108 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=96204 max_load_reduction_pct=0.744031 moved_to_replicas=152036 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[0, 65190, 0, 56833, 3, 10803, 129301, 14] avg=32768.0000 max=129301 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.945953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] avg=32768.0000 max=33097 max_rank=3 min=32108 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=96204 max_load_reduction_pct=0.744031 moved_to_replicas=152036 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] avg=32768.0000 max=33097 max_rank=3 min=31989 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37664 max_load_reduction_pct=0.532271 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[0, 65190, 0, 56833, 3, 10803, 129301, 14] avg=32768.0000 max=129301 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.945953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32766, 214, 65463, 60171, 1, 32768, 0, 70761] avg=32768.0000 max=70761 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.159454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] avg=32768.0000 max=33097 max_rank=3 min=32108 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=96204 max_load_reduction_pct=0.744031 moved_to_replicas=152036 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] avg=32768.0000 max=33097 max_rank=3 min=32108 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=96204 max_load_reduction_pct=0.744031 moved_to_replicas=152036 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 32118, 33095, 33096, 32108] avg=32768.0000 max=33097 max_rank=3 min=32108 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=96204 max_load_reduction_pct=0.744031 moved_to_replicas=152036 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32766, 214, 65463, 60171, 1, 32768, 0, 70761] avg=32768.0000 max=70761 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.159454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32766, 214, 65463, 60171, 1, 32768, 0, 70761] avg=32768.0000 max=70761 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.159454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32766, 214, 65463, 60171, 1, 32768, 0, 70761] avg=32768.0000 max=70761 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.159454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 31110, 32737, 32774, 70287, 29700] avg=32768.0000 max=70287 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.144989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] avg=32768.0000 max=33097 max_rank=3 min=31989 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37664 max_load_reduction_pct=0.532271 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] avg=32768.0000 max=33097 max_rank=3 min=31989 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37664 max_load_reduction_pct=0.532271 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] avg=32768.0000 max=33097 max_rank=3 min=31989 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37664 max_load_reduction_pct=0.532271 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32766, 214, 65463, 60171, 1, 32768, 0, 70761] avg=32768.0000 max=70761 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.159454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32766, 214, 65463, 60171, 1, 32768, 0, 70761] avg=32768.0000 max=70761 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.159454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] avg=32768.0000 max=33097 max_rank=3 min=31989 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37664 max_load_reduction_pct=0.532271 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32138 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=37191 max_load_reduction_pct=0.529131 moved_to_replicas=37191 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32766, 214, 65463, 60171, 1, 32768, 0, 70761] avg=32768.0000 max=70761 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.159454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] avg=32768.0000 max=33097 max_rank=3 min=31989 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37664 max_load_reduction_pct=0.532271 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 31110, 32737, 32774, 70287, 29700] avg=32768.0000 max=70287 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.144989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] avg=32768.0000 max=33097 max_rank=3 min=31989 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37664 max_load_reduction_pct=0.532271 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32766, 32581, 33096, 33097, 31989, 32768, 32751, 33096] avg=32768.0000 max=33097 max_rank=3 min=31989 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37664 max_load_reduction_pct=0.532271 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 31110, 32737, 32774, 70287, 29700] avg=32768.0000 max=70287 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.144989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 31110, 32737, 32774, 70287, 29700] avg=32768.0000 max=70287 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.144989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32908, 65397, 0, 65531, 32766, 32769, 32773, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 31110, 32737, 32774, 70287, 29700] avg=32768.0000 max=70287 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.144989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32138 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=37191 max_load_reduction_pct=0.529131 moved_to_replicas=37191 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32138 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=37191 max_load_reduction_pct=0.529131 moved_to_replicas=37191 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32138 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=37191 max_load_reduction_pct=0.529131 moved_to_replicas=37191 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 31110, 32737, 32774, 70287, 29700] avg=32768.0000 max=70287 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.144989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] avg=32768.0000 max=33096 max_rank=1 min=32301 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32138 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=37191 max_load_reduction_pct=0.529131 moved_to_replicas=37191 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 31110, 32737, 32774, 70287, 29700] avg=32768.0000 max=70287 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.144989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 31110, 32737, 32774, 70287, 29700] avg=32768.0000 max=70287 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.144989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32138 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=37191 max_load_reduction_pct=0.529131 moved_to_replicas=37191 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32908, 65397, 0, 65531, 32766, 32769, 32773, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32908, 65397, 0, 65531, 32766, 32769, 32773, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32908, 65397, 0, 65531, 32766, 32769, 32773, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32138 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=37191 max_load_reduction_pct=0.529131 moved_to_replicas=37191 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 32138, 32737, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32138 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=37191 max_load_reduction_pct=0.529131 moved_to_replicas=37191 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32908, 65397, 0, 65531, 32766, 32769, 32773, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] avg=32768.0000 max=33096 max_rank=1 min=32301 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] avg=32768.0000 max=33096 max_rank=1 min=32301 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] avg=32768.0000 max=33096 max_rank=1 min=32301 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32908, 65397, 0, 65531, 32766, 32769, 32773, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] avg=32768.0000 max=33096 max_rank=1 min=32301 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32908, 65397, 0, 65531, 32766, 32769, 32773, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32908, 65397, 0, 65531, 32766, 32769, 32773, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] avg=32768.0000 max=33096 max_rank=1 min=32301 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] avg=32768.0000 max=33096 max_rank=1 min=32301 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32908, 33096, 32435, 33096, 32766, 32769, 32773, 32301] avg=32768.0000 max=33096 max_rank=1 min=32301 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64736 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=10 total_expected_recv=323950 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=82944 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=10 total_expected_recv=329107 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78464 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=10 total_expected_recv=326051 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=80640 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=10 total_expected_recv=327763 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=80256 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=10 total_expected_recv=329174 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=10 total_expected_recv=328358 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77952 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=10 total_expected_recv=328578 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78208 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=10 total_expected_recv=328459 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77952 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:52:00] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 239.4 | throughput per GPU (TFLOP/s/GPU): 515.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.973934E+00 | loss scale: 1.0 | grad norm: 4.567 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=31412 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32156 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=31412 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32156 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31763 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32128 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31763 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32128 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=30802 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=30802 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=31722 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32162 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=31722 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32162 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=31005 expected_overflow=0 exact_recv_per_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=31005 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32473 expected_overflow=0 exact_recv_per_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32998 expected_overflow=0 exact_recv_per_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32473 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32998 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=31648 expected_overflow=0 exact_recv_per_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=31648 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=31808 expected_overflow=0 exact_recv_per_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=31808 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32031 expected_overflow=0 exact_recv_per_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32152 expected_overflow=0 exact_recv_per_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32031 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32152 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32235 expected_overflow=0 exact_recv_per_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32235 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[40722, 42286, 35085, 27365, 28535, 25898, 26280, 35973] avg=32768.0000 max=42286 max_rank=1 min=25898 min_rank=5 imbalance_ratio=1.290466 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[40722, 42286, 35085, 27365, 28535, 25898, 26280, 35973] avg=32768.0000 max=42286 max_rank=1 min=25898 min_rank=5 imbalance_ratio=1.290466 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[40722, 42286, 35085, 27365, 28535, 25898, 26280, 35973] avg=32768.0000 max=42286 max_rank=1 min=25898 min_rank=5 imbalance_ratio=1.290466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[40722, 42286, 35085, 27365, 28535, 25898, 26280, 35973] avg=32768.0000 max=42286 max_rank=1 min=25898 min_rank=5 imbalance_ratio=1.290466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] avg=32768.0000 max=33097 max_rank=0 min=31412 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9189 max_load_reduction_pct=0.217306 moved_to_replicas=21682 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[40722, 42286, 35085, 27365, 28535, 25898, 26280, 35973] avg=32768.0000 max=42286 max_rank=1 min=25898 min_rank=5 imbalance_ratio=1.290466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] avg=32768.0000 max=33097 max_rank=0 min=31412 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9189 max_load_reduction_pct=0.217306 moved_to_replicas=21682 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[40722, 42286, 35085, 27365, 28535, 25898, 26280, 35973] avg=32768.0000 max=42286 max_rank=1 min=25898 min_rank=5 imbalance_ratio=1.290466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] avg=32768.0000 max=33097 max_rank=0 min=31412 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9189 max_load_reduction_pct=0.217306 moved_to_replicas=21682 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[40722, 42286, 35085, 27365, 28535, 25898, 26280, 35973] avg=32768.0000 max=42286 max_rank=1 min=25898 min_rank=5 imbalance_ratio=1.290466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] avg=32768.0000 max=33097 max_rank=0 min=31412 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9189 max_load_reduction_pct=0.217306 moved_to_replicas=21682 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[40722, 42286, 35085, 27365, 28535, 25898, 26280, 35973] avg=32768.0000 max=42286 max_rank=1 min=25898 min_rank=5 imbalance_ratio=1.290466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] avg=32768.0000 max=33097 max_rank=0 min=31412 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9189 max_load_reduction_pct=0.217306 moved_to_replicas=21682 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] avg=32768.0000 max=33097 max_rank=0 min=31412 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9189 max_load_reduction_pct=0.217306 moved_to_replicas=21682 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65428, 59380, 35860, 64979, 4879, 3350, 1590, 26678] avg=32768.0000 max=65428 max_rank=0 min=1590 min_rank=6 imbalance_ratio=1.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] avg=32768.0000 max=33097 max_rank=0 min=31412 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9189 max_load_reduction_pct=0.217306 moved_to_replicas=21682 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65428, 59380, 35860, 64979, 4879, 3350, 1590, 26678] avg=32768.0000 max=65428 max_rank=0 min=1590 min_rank=6 imbalance_ratio=1.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 31412, 33095, 32156, 33096] avg=32768.0000 max=33097 max_rank=0 min=31412 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9189 max_load_reduction_pct=0.217306 moved_to_replicas=21682 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65428, 59380, 35860, 64979, 4879, 3350, 1590, 26678] avg=32768.0000 max=65428 max_rank=0 min=1590 min_rank=6 imbalance_ratio=1.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65428, 59380, 35860, 64979, 4879, 3350, 1590, 26678] avg=32768.0000 max=65428 max_rank=0 min=1590 min_rank=6 imbalance_ratio=1.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] avg=32768.0000 max=33097 max_rank=5 min=31763 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32331 max_load_reduction_pct=0.494146 moved_to_replicas=93263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65428, 59380, 35860, 64979, 4879, 3350, 1590, 26678] avg=32768.0000 max=65428 max_rank=0 min=1590 min_rank=6 imbalance_ratio=1.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65428, 59380, 35860, 64979, 4879, 3350, 1590, 26678] avg=32768.0000 max=65428 max_rank=0 min=1590 min_rank=6 imbalance_ratio=1.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] avg=32768.0000 max=33097 max_rank=5 min=31763 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32331 max_load_reduction_pct=0.494146 moved_to_replicas=93263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] avg=32768.0000 max=33097 max_rank=5 min=31763 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32331 max_load_reduction_pct=0.494146 moved_to_replicas=93263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65428, 59380, 35860, 64979, 4879, 3350, 1590, 26678] avg=32768.0000 max=65428 max_rank=0 min=1590 min_rank=6 imbalance_ratio=1.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] avg=32768.0000 max=33097 max_rank=5 min=31763 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32331 max_load_reduction_pct=0.494146 moved_to_replicas=93263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65428, 59380, 35860, 64979, 4879, 3350, 1590, 26678] avg=32768.0000 max=65428 max_rank=0 min=1590 min_rank=6 imbalance_ratio=1.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] avg=32768.0000 max=33097 max_rank=5 min=31763 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32331 max_load_reduction_pct=0.494146 moved_to_replicas=93263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] avg=32768.0000 max=33097 max_rank=5 min=31763 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32331 max_load_reduction_pct=0.494146 moved_to_replicas=93263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[49655, 45798, 36867, 87609, 7561, 5068, 2, 29584] avg=32768.0000 max=87609 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.673615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] avg=32768.0000 max=33097 max_rank=5 min=31763 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32331 max_load_reduction_pct=0.494146 moved_to_replicas=93263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[49655, 45798, 36867, 87609, 7561, 5068, 2, 29584] avg=32768.0000 max=87609 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.673615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31763, 33097, 32128, 32773] avg=32768.0000 max=33097 max_rank=5 min=31763 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32331 max_load_reduction_pct=0.494146 moved_to_replicas=93263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[49655, 45798, 36867, 87609, 7561, 5068, 2, 29584] avg=32768.0000 max=87609 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.673615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] avg=32768.0000 max=33097 max_rank=0 min=30802 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54512 max_load_reduction_pct=0.622219 moved_to_replicas=87545 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[49655, 45798, 36867, 87609, 7561, 5068, 2, 29584] avg=32768.0000 max=87609 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.673615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[49655, 45798, 36867, 87609, 7561, 5068, 2, 29584] avg=32768.0000 max=87609 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.673615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[49655, 45798, 36867, 87609, 7561, 5068, 2, 29584] avg=32768.0000 max=87609 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.673615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] avg=32768.0000 max=33097 max_rank=0 min=30802 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54512 max_load_reduction_pct=0.622219 moved_to_replicas=87545 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] avg=32768.0000 max=33097 max_rank=0 min=30802 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54512 max_load_reduction_pct=0.622219 moved_to_replicas=87545 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[49655, 45798, 36867, 87609, 7561, 5068, 2, 29584] avg=32768.0000 max=87609 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.673615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] avg=32768.0000 max=33097 max_rank=0 min=30802 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54512 max_load_reduction_pct=0.622219 moved_to_replicas=87545 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[49655, 45798, 36867, 87609, 7561, 5068, 2, 29584] avg=32768.0000 max=87609 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.673615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] avg=32768.0000 max=33097 max_rank=0 min=30802 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54512 max_load_reduction_pct=0.622219 moved_to_replicas=87545 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] avg=32768.0000 max=33097 max_rank=0 min=30802 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54512 max_load_reduction_pct=0.622219 moved_to_replicas=87545 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[39963, 4287, 31390, 16322, 31722, 31, 11904, 126525] avg=32768.0000 max=126525 max_rank=7 min=31 min_rank=5 imbalance_ratio=3.861237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] avg=32768.0000 max=33097 max_rank=0 min=30802 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54512 max_load_reduction_pct=0.622219 moved_to_replicas=87545 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[39963, 4287, 31390, 16322, 31722, 31, 11904, 126525] avg=32768.0000 max=126525 max_rank=7 min=31 min_rank=5 imbalance_ratio=3.861237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[39963, 4287, 31390, 16322, 31722, 31, 11904, 126525] avg=32768.0000 max=126525 max_rank=7 min=31 min_rank=5 imbalance_ratio=3.861237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 30802, 32766, 33096] avg=32768.0000 max=33097 max_rank=0 min=30802 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54512 max_load_reduction_pct=0.622219 moved_to_replicas=87545 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31722 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=93428 max_load_reduction_pct=0.738415 moved_to_replicas=100296 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[39963, 4287, 31390, 16322, 31722, 31, 11904, 126525] avg=32768.0000 max=126525 max_rank=7 min=31 min_rank=5 imbalance_ratio=3.861237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[39963, 4287, 31390, 16322, 31722, 31, 11904, 126525] avg=32768.0000 max=126525 max_rank=7 min=31 min_rank=5 imbalance_ratio=3.861237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[39963, 4287, 31390, 16322, 31722, 31, 11904, 126525] avg=32768.0000 max=126525 max_rank=7 min=31 min_rank=5 imbalance_ratio=3.861237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31722 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=93428 max_load_reduction_pct=0.738415 moved_to_replicas=100296 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31722 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=93428 max_load_reduction_pct=0.738415 moved_to_replicas=100296 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[39963, 4287, 31390, 16322, 31722, 31, 11904, 126525] avg=32768.0000 max=126525 max_rank=7 min=31 min_rank=5 imbalance_ratio=3.861237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31722 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=93428 max_load_reduction_pct=0.738415 moved_to_replicas=100296 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[39963, 4287, 31390, 16322, 31722, 31, 11904, 126525] avg=32768.0000 max=126525 max_rank=7 min=31 min_rank=5 imbalance_ratio=3.861237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31722 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=93428 max_load_reduction_pct=0.738415 moved_to_replicas=100296 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[13571, 1501, 5, 32887, 98223, 45212, 35475, 35270] avg=32768.0000 max=98223 max_rank=4 min=5 min_rank=2 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31722 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=93428 max_load_reduction_pct=0.738415 moved_to_replicas=100296 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31722 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=93428 max_load_reduction_pct=0.738415 moved_to_replicas=100296 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[13571, 1501, 5, 32887, 98223, 45212, 35475, 35270] avg=32768.0000 max=98223 max_rank=4 min=5 min_rank=2 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[33095, 33096, 32162, 33096, 31722, 32780, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31722 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=93428 max_load_reduction_pct=0.738415 moved_to_replicas=100296 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[13571, 1501, 5, 32887, 98223, 45212, 35475, 35270] avg=32768.0000 max=98223 max_rank=4 min=5 min_rank=2 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31005 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=81796 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[13571, 1501, 5, 32887, 98223, 45212, 35475, 35270] avg=32768.0000 max=98223 max_rank=4 min=5 min_rank=2 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[13571, 1501, 5, 32887, 98223, 45212, 35475, 35270] avg=32768.0000 max=98223 max_rank=4 min=5 min_rank=2 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[13571, 1501, 5, 32887, 98223, 45212, 35475, 35270] avg=32768.0000 max=98223 max_rank=4 min=5 min_rank=2 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31005 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=81796 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31005 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=81796 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[13571, 1501, 5, 32887, 98223, 45212, 35475, 35270] avg=32768.0000 max=98223 max_rank=4 min=5 min_rank=2 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31005 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=81796 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31005 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=81796 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[13571, 1501, 5, 32887, 98223, 45212, 35475, 35270] avg=32768.0000 max=98223 max_rank=4 min=5 min_rank=2 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31005 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=81796 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31005 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=81796 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[31005, 33096, 32772, 32887, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31005 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=81796 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[4, 26820, 16678, 35095, 5, 37922, 112852, 32768] avg=32768.0000 max=112852 max_rank=6 min=4 min_rank=0 imbalance_ratio=3.443970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[4, 26820, 16678, 35095, 5, 37922, 112852, 32768] avg=32768.0000 max=112852 max_rank=6 min=4 min_rank=0 imbalance_ratio=3.443970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[4, 26820, 16678, 35095, 5, 37922, 112852, 32768] avg=32768.0000 max=112852 max_rank=6 min=4 min_rank=0 imbalance_ratio=3.443970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32473, 32763, 32768, 32768, 32768, 32730, 32998, 32876] avg=32768.0000 max=32998 max_rank=6 min=32473 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=79756 max_load_reduction_pct=0.706731 moved_to_replicas=86581 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[4, 26820, 16678, 35095, 5, 37922, 112852, 32768] avg=32768.0000 max=112852 max_rank=6 min=4 min_rank=0 imbalance_ratio=3.443970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[4, 26820, 16678, 35095, 5, 37922, 112852, 32768] avg=32768.0000 max=112852 max_rank=6 min=4 min_rank=0 imbalance_ratio=3.443970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=79756 max_load_reduction_pct=0.706731 moved_to_replicas=86581 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[4, 26820, 16678, 35095, 5, 37922, 112852, 32768] avg=32768.0000 max=112852 max_rank=6 min=4 min_rank=0 imbalance_ratio=3.443970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=79756 max_load_reduction_pct=0.706731 moved_to_replicas=86581 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[4, 26820, 16678, 35095, 5, 37922, 112852, 32768] avg=32768.0000 max=112852 max_rank=6 min=4 min_rank=0 imbalance_ratio=3.443970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=79756 max_load_reduction_pct=0.706731 moved_to_replicas=86581 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[4, 26820, 16678, 35095, 5, 37922, 112852, 32768] avg=32768.0000 max=112852 max_rank=6 min=4 min_rank=0 imbalance_ratio=3.443970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=79756 max_load_reduction_pct=0.706731 moved_to_replicas=86581 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32750, 51492, 32771, 78944, 300, 33128, 6, 32753] avg=32768.0000 max=78944 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.409180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=79756 max_load_reduction_pct=0.706731 moved_to_replicas=86581 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=79756 max_load_reduction_pct=0.706731 moved_to_replicas=86581 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32750, 51492, 32771, 78944, 300, 33128, 6, 32753] avg=32768.0000 max=78944 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.409180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32750, 51492, 32771, 78944, 300, 33128, 6, 32753] avg=32768.0000 max=78944 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.409180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32771, 31648, 32916, 33096, 32755, 33094, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=79756 max_load_reduction_pct=0.706731 moved_to_replicas=86581 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] avg=32768.0000 max=33097 max_rank=3 min=31808 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=45847 max_load_reduction_pct=0.580753 moved_to_replicas=64276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32750, 51492, 32771, 78944, 300, 33128, 6, 32753] avg=32768.0000 max=78944 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.409180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32750, 51492, 32771, 78944, 300, 33128, 6, 32753] avg=32768.0000 max=78944 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.409180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] avg=32768.0000 max=33097 max_rank=3 min=31808 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=45847 max_load_reduction_pct=0.580753 moved_to_replicas=64276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] avg=32768.0000 max=33097 max_rank=3 min=31808 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=45847 max_load_reduction_pct=0.580753 moved_to_replicas=64276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32750, 51492, 32771, 78944, 300, 33128, 6, 32753] avg=32768.0000 max=78944 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.409180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32750, 51492, 32771, 78944, 300, 33128, 6, 32753] avg=32768.0000 max=78944 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.409180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] avg=32768.0000 max=33097 max_rank=3 min=31808 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=45847 max_load_reduction_pct=0.580753 moved_to_replicas=64276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32750, 51492, 32771, 78944, 300, 33128, 6, 32753] avg=32768.0000 max=78944 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.409180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] avg=32768.0000 max=33097 max_rank=3 min=31808 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=45847 max_load_reduction_pct=0.580753 moved_to_replicas=64276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[60648, 37, 32771, 5, 0, 36413, 66086, 66184] avg=32768.0000 max=66184 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.019775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] avg=32768.0000 max=33097 max_rank=3 min=31808 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=45847 max_load_reduction_pct=0.580753 moved_to_replicas=64276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] avg=32768.0000 max=33097 max_rank=3 min=31808 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=45847 max_load_reduction_pct=0.580753 moved_to_replicas=64276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[60648, 37, 32771, 5, 0, 36413, 66086, 66184] avg=32768.0000 max=66184 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.019775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[60648, 37, 32771, 5, 0, 36413, 66086, 66184] avg=32768.0000 max=66184 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.019775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32750, 33096, 32771, 33097, 31808, 33096, 32773, 32753] avg=32768.0000 max=33097 max_rank=3 min=31808 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=45847 max_load_reduction_pct=0.580753 moved_to_replicas=64276 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=32031 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=33086 max_load_reduction_pct=0.499909 moved_to_replicas=96947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[60648, 37, 32771, 5, 0, 36413, 66086, 66184] avg=32768.0000 max=66184 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.019775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[60648, 37, 32771, 5, 0, 36413, 66086, 66184] avg=32768.0000 max=66184 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.019775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=32031 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=33086 max_load_reduction_pct=0.499909 moved_to_replicas=96947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=32031 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=33086 max_load_reduction_pct=0.499909 moved_to_replicas=96947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[60648, 37, 32771, 5, 0, 36413, 66086, 66184] avg=32768.0000 max=66184 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.019775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[60648, 37, 32771, 5, 0, 36413, 66086, 66184] avg=32768.0000 max=66184 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.019775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=32031 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=33086 max_load_reduction_pct=0.499909 moved_to_replicas=96947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[60648, 37, 32771, 5, 0, 36413, 66086, 66184] avg=32768.0000 max=66184 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.019775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=32031 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=33086 max_load_reduction_pct=0.499909 moved_to_replicas=96947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33132, 32622, 32759, 32806, 32759, 32771, 65295, 0] avg=32768.0000 max=65295 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.992645 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=32031 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=33086 max_load_reduction_pct=0.499909 moved_to_replicas=96947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=32031 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=33086 max_load_reduction_pct=0.499909 moved_to_replicas=96947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33132, 32622, 32759, 32806, 32759, 32771, 65295, 0] avg=32768.0000 max=65295 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.992645 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33132, 32622, 32759, 32806, 32759, 32771, 65295, 0] avg=32768.0000 max=65295 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.992645 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32804, 32771, 32031, 32152, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=32031 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=33086 max_load_reduction_pct=0.499909 moved_to_replicas=96947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] avg=32768.0000 max=33096 max_rank=0 min=32235 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32199 max_load_reduction_pct=0.493131 moved_to_replicas=32235 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33132, 32622, 32759, 32806, 32759, 32771, 65295, 0] avg=32768.0000 max=65295 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.992645 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33132, 32622, 32759, 32806, 32759, 32771, 65295, 0] avg=32768.0000 max=65295 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.992645 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] avg=32768.0000 max=33096 max_rank=0 min=32235 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32199 max_load_reduction_pct=0.493131 moved_to_replicas=32235 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] avg=32768.0000 max=33096 max_rank=0 min=32235 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32199 max_load_reduction_pct=0.493131 moved_to_replicas=32235 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33132, 32622, 32759, 32806, 32759, 32771, 65295, 0] avg=32768.0000 max=65295 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.992645 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33132, 32622, 32759, 32806, 32759, 32771, 65295, 0] avg=32768.0000 max=65295 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.992645 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] avg=32768.0000 max=33096 max_rank=0 min=32235 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32199 max_load_reduction_pct=0.493131 moved_to_replicas=32235 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33132, 32622, 32759, 32806, 32759, 32771, 65295, 0] avg=32768.0000 max=65295 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.992645 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] avg=32768.0000 max=33096 max_rank=0 min=32235 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32199 max_load_reduction_pct=0.493131 moved_to_replicas=32235 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] avg=32768.0000 max=33096 max_rank=0 min=32235 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32199 max_load_reduction_pct=0.493131 moved_to_replicas=32235 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] avg=32768.0000 max=33096 max_rank=0 min=32235 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32199 max_load_reduction_pct=0.493131 moved_to_replicas=32235 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32622, 32759, 32806, 32759, 32771, 33096, 32235] avg=32768.0000 max=33096 max_rank=0 min=32235 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32199 max_load_reduction_pct=0.493131 moved_to_replicas=32235 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=10 total_expected_recv=329068 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=10 total_expected_recv=328885 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=10 total_expected_recv=323330 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=83456 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=10 total_expected_recv=327578 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79616 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=10 total_expected_recv=328207 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78976 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=10 total_expected_recv=327657 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78208 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=10 total_expected_recv=328413 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78720 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=10 total_expected_recv=328302 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78336 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:52:01] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 240.3 | throughput per GPU (TFLOP/s/GPU): 513.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.759029E+00 | loss scale: 1.0 | grad norm: 4.313 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31775 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31792 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31775 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31792 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=31744 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=31824 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=31744 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=31824 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=31468 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32100 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=31468 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32100 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=31326 expected_overflow=0 exact_recv_per_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=31326 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=31133 expected_overflow=0 exact_recv_per_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=31133 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=31993 expected_overflow=0 exact_recv_per_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=31993 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32498 expected_overflow=0 exact_recv_per_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=31873 expected_overflow=0 exact_recv_per_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32498 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=31873 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=31926 expected_overflow=0 exact_recv_per_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=31926 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=31460 expected_overflow=0 exact_recv_per_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=31460 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=31490 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=31490 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[39255, 40786, 33306, 28089, 33645, 23343, 27240, 36480] avg=32768.0000 max=40786 max_rank=1 min=23343 min_rank=5 imbalance_ratio=1.244690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[39255, 40786, 33306, 28089, 33645, 23343, 27240, 36480] avg=32768.0000 max=40786 max_rank=1 min=23343 min_rank=5 imbalance_ratio=1.244690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[39255, 40786, 33306, 28089, 33645, 23343, 27240, 36480] avg=32768.0000 max=40786 max_rank=1 min=23343 min_rank=5 imbalance_ratio=1.244690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[39255, 40786, 33306, 28089, 33645, 23343, 27240, 36480] avg=32768.0000 max=40786 max_rank=1 min=23343 min_rank=5 imbalance_ratio=1.244690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7689 max_load_reduction_pct=0.188521 moved_to_replicas=17992 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7689 max_load_reduction_pct=0.188521 moved_to_replicas=17992 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7689 max_load_reduction_pct=0.188521 moved_to_replicas=17992 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[39255, 40786, 33306, 28089, 33645, 23343, 27240, 36480] avg=32768.0000 max=40786 max_rank=1 min=23343 min_rank=5 imbalance_ratio=1.244690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7689 max_load_reduction_pct=0.188521 moved_to_replicas=17992 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[39255, 40786, 33306, 28089, 33645, 23343, 27240, 36480] avg=32768.0000 max=40786 max_rank=1 min=23343 min_rank=5 imbalance_ratio=1.244690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[39255, 40786, 33306, 28089, 33645, 23343, 27240, 36480] avg=32768.0000 max=40786 max_rank=1 min=23343 min_rank=5 imbalance_ratio=1.244690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[39255, 40786, 33306, 28089, 33645, 23343, 27240, 36480] avg=32768.0000 max=40786 max_rank=1 min=23343 min_rank=5 imbalance_ratio=1.244690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[57175, 70492, 31158, 42280, 18849, 7974, 14524, 19692] avg=32768.0000 max=70492 max_rank=1 min=7974 min_rank=5 imbalance_ratio=2.151245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7689 max_load_reduction_pct=0.188521 moved_to_replicas=17992 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[57175, 70492, 31158, 42280, 18849, 7974, 14524, 19692] avg=32768.0000 max=70492 max_rank=1 min=7974 min_rank=5 imbalance_ratio=2.151245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[57175, 70492, 31158, 42280, 18849, 7974, 14524, 19692] avg=32768.0000 max=70492 max_rank=1 min=7974 min_rank=5 imbalance_ratio=2.151245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7689 max_load_reduction_pct=0.188521 moved_to_replicas=17992 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7689 max_load_reduction_pct=0.188521 moved_to_replicas=17992 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[33097, 33096, 33096, 31775, 33096, 31792, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7689 max_load_reduction_pct=0.188521 moved_to_replicas=17992 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[57175, 70492, 31158, 42280, 18849, 7974, 14524, 19692] avg=32768.0000 max=70492 max_rank=1 min=7974 min_rank=5 imbalance_ratio=2.151245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31744 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=37395 max_load_reduction_pct=0.530486 moved_to_replicas=70659 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31744 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=37395 max_load_reduction_pct=0.530486 moved_to_replicas=70659 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31744 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=37395 max_load_reduction_pct=0.530486 moved_to_replicas=70659 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[57175, 70492, 31158, 42280, 18849, 7974, 14524, 19692] avg=32768.0000 max=70492 max_rank=1 min=7974 min_rank=5 imbalance_ratio=2.151245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31744 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=37395 max_load_reduction_pct=0.530486 moved_to_replicas=70659 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[57175, 70492, 31158, 42280, 18849, 7974, 14524, 19692] avg=32768.0000 max=70492 max_rank=1 min=7974 min_rank=5 imbalance_ratio=2.151245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[57175, 70492, 31158, 42280, 18849, 7974, 14524, 19692] avg=32768.0000 max=70492 max_rank=1 min=7974 min_rank=5 imbalance_ratio=2.151245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[57175, 70492, 31158, 42280, 18849, 7974, 14524, 19692] avg=32768.0000 max=70492 max_rank=1 min=7974 min_rank=5 imbalance_ratio=2.151245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[16921, 33583, 64703, 75590, 36001, 10727, 9, 24610] avg=32768.0000 max=75590 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.306824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[16921, 33583, 64703, 75590, 36001, 10727, 9, 24610] avg=32768.0000 max=75590 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.306824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31744 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=37395 max_load_reduction_pct=0.530486 moved_to_replicas=70659 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[16921, 33583, 64703, 75590, 36001, 10727, 9, 24610] avg=32768.0000 max=75590 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.306824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31744 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=37395 max_load_reduction_pct=0.530486 moved_to_replicas=70659 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31744 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=37395 max_load_reduction_pct=0.530486 moved_to_replicas=70659 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 31824, 33096, 33095, 31744, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31744 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=37395 max_load_reduction_pct=0.530486 moved_to_replicas=70659 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[16921, 33583, 64703, 75590, 36001, 10727, 9, 24610] avg=32768.0000 max=75590 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.306824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] avg=32768.0000 max=33098 max_rank=5 min=31468 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=42492 max_load_reduction_pct=0.562138 moved_to_replicas=77493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] avg=32768.0000 max=33098 max_rank=5 min=31468 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=42492 max_load_reduction_pct=0.562138 moved_to_replicas=77493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] avg=32768.0000 max=33098 max_rank=5 min=31468 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=42492 max_load_reduction_pct=0.562138 moved_to_replicas=77493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[16921, 33583, 64703, 75590, 36001, 10727, 9, 24610] avg=32768.0000 max=75590 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.306824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] avg=32768.0000 max=33098 max_rank=5 min=31468 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=42492 max_load_reduction_pct=0.562138 moved_to_replicas=77493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[16921, 33583, 64703, 75590, 36001, 10727, 9, 24610] avg=32768.0000 max=75590 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.306824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[16921, 33583, 64703, 75590, 36001, 10727, 9, 24610] avg=32768.0000 max=75590 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.306824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[16921, 33583, 64703, 75590, 36001, 10727, 9, 24610] avg=32768.0000 max=75590 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.306824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[59369, 1398, 26110, 28343, 15852, 141, 12874, 118057] avg=32768.0000 max=118057 max_rank=7 min=141 min_rank=5 imbalance_ratio=3.602814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[59369, 1398, 26110, 28343, 15852, 141, 12874, 118057] avg=32768.0000 max=118057 max_rank=7 min=141 min_rank=5 imbalance_ratio=3.602814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] avg=32768.0000 max=33098 max_rank=5 min=31468 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=42492 max_load_reduction_pct=0.562138 moved_to_replicas=77493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[59369, 1398, 26110, 28343, 15852, 141, 12874, 118057] avg=32768.0000 max=118057 max_rank=7 min=141 min_rank=5 imbalance_ratio=3.602814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] avg=32768.0000 max=33098 max_rank=5 min=31468 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=42492 max_load_reduction_pct=0.562138 moved_to_replicas=77493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] avg=32768.0000 max=33098 max_rank=5 min=31468 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=42492 max_load_reduction_pct=0.562138 moved_to_replicas=77493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 33096, 33098, 32100, 31468] avg=32768.0000 max=33098 max_rank=5 min=31468 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=42492 max_load_reduction_pct=0.562138 moved_to_replicas=77493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[59369, 1398, 26110, 28343, 15852, 141, 12874, 118057] avg=32768.0000 max=118057 max_rank=7 min=141 min_rank=5 imbalance_ratio=3.602814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31326 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=84960 max_load_reduction_pct=0.719652 moved_to_replicas=111234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31326 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=84960 max_load_reduction_pct=0.719652 moved_to_replicas=111234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31326 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=84960 max_load_reduction_pct=0.719652 moved_to_replicas=111234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[59369, 1398, 26110, 28343, 15852, 141, 12874, 118057] avg=32768.0000 max=118057 max_rank=7 min=141 min_rank=5 imbalance_ratio=3.602814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31326 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=84960 max_load_reduction_pct=0.719652 moved_to_replicas=111234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[59369, 1398, 26110, 28343, 15852, 141, 12874, 118057] avg=32768.0000 max=118057 max_rank=7 min=141 min_rank=5 imbalance_ratio=3.602814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[59369, 1398, 26110, 28343, 15852, 141, 12874, 118057] avg=32768.0000 max=118057 max_rank=7 min=141 min_rank=5 imbalance_ratio=3.602814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[59369, 1398, 26110, 28343, 15852, 141, 12874, 118057] avg=32768.0000 max=118057 max_rank=7 min=141 min_rank=5 imbalance_ratio=3.602814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[8210, 33, 9, 15881, 103824, 85153, 47459, 1575] avg=32768.0000 max=103824 max_rank=4 min=9 min_rank=2 imbalance_ratio=3.168457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[8210, 33, 9, 15881, 103824, 85153, 47459, 1575] avg=32768.0000 max=103824 max_rank=4 min=9 min_rank=2 imbalance_ratio=3.168457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31326 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=84960 max_load_reduction_pct=0.719652 moved_to_replicas=111234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[8210, 33, 9, 15881, 103824, 85153, 47459, 1575] avg=32768.0000 max=103824 max_rank=4 min=9 min_rank=2 imbalance_ratio=3.168457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31326 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=84960 max_load_reduction_pct=0.719652 moved_to_replicas=111234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31326 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=84960 max_load_reduction_pct=0.719652 moved_to_replicas=111234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[33097, 32458, 33096, 31326, 33095, 32880, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31326 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=84960 max_load_reduction_pct=0.719652 moved_to_replicas=111234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31133 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=70727 max_load_reduction_pct=0.681220 moved_to_replicas=137148 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31133 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=70727 max_load_reduction_pct=0.681220 moved_to_replicas=137148 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[8210, 33, 9, 15881, 103824, 85153, 47459, 1575] avg=32768.0000 max=103824 max_rank=4 min=9 min_rank=2 imbalance_ratio=3.168457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31133 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=70727 max_load_reduction_pct=0.681220 moved_to_replicas=137148 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[8210, 33, 9, 15881, 103824, 85153, 47459, 1575] avg=32768.0000 max=103824 max_rank=4 min=9 min_rank=2 imbalance_ratio=3.168457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[8210, 33, 9, 15881, 103824, 85153, 47459, 1575] avg=32768.0000 max=103824 max_rank=4 min=9 min_rank=2 imbalance_ratio=3.168457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31133 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=70727 max_load_reduction_pct=0.681220 moved_to_replicas=137148 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[8210, 33, 9, 15881, 103824, 85153, 47459, 1575] avg=32768.0000 max=103824 max_rank=4 min=9 min_rank=2 imbalance_ratio=3.168457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33622, 32708, 32742, 32771, 31429, 32642, 64580, 1650] avg=32768.0000 max=64580 max_rank=6 min=1650 min_rank=7 imbalance_ratio=1.970825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33622, 32708, 32742, 32771, 31429, 32642, 64580, 1650] avg=32768.0000 max=64580 max_rank=6 min=1650 min_rank=7 imbalance_ratio=1.970825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[8210, 33, 9, 15881, 103824, 85153, 47459, 1575] avg=32768.0000 max=103824 max_rank=4 min=9 min_rank=2 imbalance_ratio=3.168457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31133 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=70727 max_load_reduction_pct=0.681220 moved_to_replicas=137148 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33622, 32708, 32742, 32771, 31429, 32642, 64580, 1650] avg=32768.0000 max=64580 max_rank=6 min=1650 min_rank=7 imbalance_ratio=1.970825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31133 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=70727 max_load_reduction_pct=0.681220 moved_to_replicas=137148 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31133 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=70727 max_load_reduction_pct=0.681220 moved_to_replicas=137148 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31993 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31484 max_load_reduction_pct=0.487519 moved_to_replicas=32010 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31993 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31484 max_load_reduction_pct=0.487519 moved_to_replicas=32010 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[33095, 32754, 32776, 31133, 33095, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31133 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=70727 max_load_reduction_pct=0.681220 moved_to_replicas=137148 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33622, 32708, 32742, 32771, 31429, 32642, 64580, 1650] avg=32768.0000 max=64580 max_rank=6 min=1650 min_rank=7 imbalance_ratio=1.970825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31993 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31484 max_load_reduction_pct=0.487519 moved_to_replicas=32010 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33622, 32708, 32742, 32771, 31429, 32642, 64580, 1650] avg=32768.0000 max=64580 max_rank=6 min=1650 min_rank=7 imbalance_ratio=1.970825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33622, 32708, 32742, 32771, 31429, 32642, 64580, 1650] avg=32768.0000 max=64580 max_rank=6 min=1650 min_rank=7 imbalance_ratio=1.970825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31993 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31484 max_load_reduction_pct=0.487519 moved_to_replicas=32010 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33622, 32708, 32742, 32771, 31429, 32642, 64580, 1650] avg=32768.0000 max=64580 max_rank=6 min=1650 min_rank=7 imbalance_ratio=1.970825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[51, 66, 3476, 32955, 0, 47783, 145050, 32763] avg=32768.0000 max=145050 max_rank=6 min=0 min_rank=4 imbalance_ratio=4.426575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[51, 66, 3476, 32955, 0, 47783, 145050, 32763] avg=32768.0000 max=145050 max_rank=6 min=0 min_rank=4 imbalance_ratio=4.426575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33622, 32708, 32742, 32771, 31429, 32642, 64580, 1650] avg=32768.0000 max=64580 max_rank=6 min=1650 min_rank=7 imbalance_ratio=1.970825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31993 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31484 max_load_reduction_pct=0.487519 moved_to_replicas=32010 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31993 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31484 max_load_reduction_pct=0.487519 moved_to_replicas=32010 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[51, 66, 3476, 32955, 0, 47783, 145050, 32763] avg=32768.0000 max=145050 max_rank=6 min=0 min_rank=4 imbalance_ratio=4.426575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31993 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31484 max_load_reduction_pct=0.487519 moved_to_replicas=32010 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] avg=32768.0000 max=33097 max_rank=2 min=31873 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=111953 max_load_reduction_pct=0.771824 moved_to_replicas=126641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32708, 32742, 32771, 31993, 32642, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31993 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31484 max_load_reduction_pct=0.487519 moved_to_replicas=32010 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] avg=32768.0000 max=33097 max_rank=2 min=31873 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=111953 max_load_reduction_pct=0.771824 moved_to_replicas=126641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[51, 66, 3476, 32955, 0, 47783, 145050, 32763] avg=32768.0000 max=145050 max_rank=6 min=0 min_rank=4 imbalance_ratio=4.426575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] avg=32768.0000 max=33097 max_rank=2 min=31873 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=111953 max_load_reduction_pct=0.771824 moved_to_replicas=126641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[51, 66, 3476, 32955, 0, 47783, 145050, 32763] avg=32768.0000 max=145050 max_rank=6 min=0 min_rank=4 imbalance_ratio=4.426575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[51, 66, 3476, 32955, 0, 47783, 145050, 32763] avg=32768.0000 max=145050 max_rank=6 min=0 min_rank=4 imbalance_ratio=4.426575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[51, 66, 3476, 32955, 0, 47783, 145050, 32763] avg=32768.0000 max=145050 max_rank=6 min=0 min_rank=4 imbalance_ratio=4.426575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] avg=32768.0000 max=33097 max_rank=2 min=31873 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=111953 max_load_reduction_pct=0.771824 moved_to_replicas=126641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32686, 49205, 32537, 56615, 32611, 53488, 115, 4887] avg=32768.0000 max=56615 max_rank=3 min=115 min_rank=6 imbalance_ratio=1.727753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32686, 49205, 32537, 56615, 32611, 53488, 115, 4887] avg=32768.0000 max=56615 max_rank=3 min=115 min_rank=6 imbalance_ratio=1.727753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[51, 66, 3476, 32955, 0, 47783, 145050, 32763] avg=32768.0000 max=145050 max_rank=6 min=0 min_rank=4 imbalance_ratio=4.426575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] avg=32768.0000 max=33097 max_rank=2 min=31873 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=111953 max_load_reduction_pct=0.771824 moved_to_replicas=126641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] avg=32768.0000 max=33097 max_rank=2 min=31873 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=111953 max_load_reduction_pct=0.771824 moved_to_replicas=126641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32686, 49205, 32537, 56615, 32611, 53488, 115, 4887] avg=32768.0000 max=56615 max_rank=3 min=115 min_rank=6 imbalance_ratio=1.727753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] avg=32768.0000 max=33097 max_rank=2 min=31873 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=111953 max_load_reduction_pct=0.771824 moved_to_replicas=126641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] avg=32768.0000 max=33097 max_rank=3 min=31926 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23518 max_load_reduction_pct=0.415402 moved_to_replicas=60020 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] avg=32768.0000 max=33097 max_rank=3 min=31926 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23518 max_load_reduction_pct=0.415402 moved_to_replicas=60020 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32498, 31873, 33097, 32955, 32767, 33094, 33097, 32763] avg=32768.0000 max=33097 max_rank=2 min=31873 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=111953 max_load_reduction_pct=0.771824 moved_to_replicas=126641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32686, 49205, 32537, 56615, 32611, 53488, 115, 4887] avg=32768.0000 max=56615 max_rank=3 min=115 min_rank=6 imbalance_ratio=1.727753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] avg=32768.0000 max=33097 max_rank=3 min=31926 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23518 max_load_reduction_pct=0.415402 moved_to_replicas=60020 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32686, 49205, 32537, 56615, 32611, 53488, 115, 4887] avg=32768.0000 max=56615 max_rank=3 min=115 min_rank=6 imbalance_ratio=1.727753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32686, 49205, 32537, 56615, 32611, 53488, 115, 4887] avg=32768.0000 max=56615 max_rank=3 min=115 min_rank=6 imbalance_ratio=1.727753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32686, 49205, 32537, 56615, 32611, 53488, 115, 4887] avg=32768.0000 max=56615 max_rank=3 min=115 min_rank=6 imbalance_ratio=1.727753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] avg=32768.0000 max=33097 max_rank=3 min=31926 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23518 max_load_reduction_pct=0.415402 moved_to_replicas=60020 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[4728, 32763, 32771, 16909, 1, 73041, 65895, 36036] avg=32768.0000 max=73041 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.229034 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[4728, 32763, 32771, 16909, 1, 73041, 65895, 36036] avg=32768.0000 max=73041 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.229034 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32686, 49205, 32537, 56615, 32611, 53488, 115, 4887] avg=32768.0000 max=56615 max_rank=3 min=115 min_rank=6 imbalance_ratio=1.727753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] avg=32768.0000 max=33097 max_rank=3 min=31926 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23518 max_load_reduction_pct=0.415402 moved_to_replicas=60020 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] avg=32768.0000 max=33097 max_rank=3 min=31926 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23518 max_load_reduction_pct=0.415402 moved_to_replicas=60020 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[4728, 32763, 32771, 16909, 1, 73041, 65895, 36036] avg=32768.0000 max=73041 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.229034 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] avg=32768.0000 max=33097 max_rank=3 min=31926 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23518 max_load_reduction_pct=0.415402 moved_to_replicas=60020 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31460 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39944 max_load_reduction_pct=0.546871 moved_to_replicas=75684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31460 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39944 max_load_reduction_pct=0.546871 moved_to_replicas=75684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32686, 33096, 32537, 33097, 32611, 33096, 33095, 31926] avg=32768.0000 max=33097 max_rank=3 min=31926 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23518 max_load_reduction_pct=0.415402 moved_to_replicas=60020 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[4728, 32763, 32771, 16909, 1, 73041, 65895, 36036] avg=32768.0000 max=73041 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.229034 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31460 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39944 max_load_reduction_pct=0.546871 moved_to_replicas=75684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[4728, 32763, 32771, 16909, 1, 73041, 65895, 36036] avg=32768.0000 max=73041 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.229034 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[4728, 32763, 32771, 16909, 1, 73041, 65895, 36036] avg=32768.0000 max=73041 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.229034 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31460 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39944 max_load_reduction_pct=0.546871 moved_to_replicas=75684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[4728, 32763, 32771, 16909, 1, 73041, 65895, 36036] avg=32768.0000 max=73041 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.229034 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[60741, 26314, 32765, 33108, 32720, 32784, 43711, 1] avg=32768.0000 max=60741 max_rank=0 min=1 min_rank=7 imbalance_ratio=1.853668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[60741, 26314, 32765, 33108, 32720, 32784, 43711, 1] avg=32768.0000 max=60741 max_rank=0 min=1 min_rank=7 imbalance_ratio=1.853668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31460 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39944 max_load_reduction_pct=0.546871 moved_to_replicas=75684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[4728, 32763, 32771, 16909, 1, 73041, 65895, 36036] avg=32768.0000 max=73041 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.229034 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31460 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39944 max_load_reduction_pct=0.546871 moved_to_replicas=75684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[60741, 26314, 32765, 33108, 32720, 32784, 43711, 1] avg=32768.0000 max=60741 max_rank=0 min=1 min_rank=7 imbalance_ratio=1.853668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31460 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39944 max_load_reduction_pct=0.546871 moved_to_replicas=75684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] avg=32768.0000 max=33098 max_rank=6 min=31490 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=27643 max_load_reduction_pct=0.455096 moved_to_replicas=38272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] avg=32768.0000 max=33098 max_rank=6 min=31490 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=27643 max_load_reduction_pct=0.455096 moved_to_replicas=38272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[60741, 26314, 32765, 33108, 32720, 32784, 43711, 1] avg=32768.0000 max=60741 max_rank=0 min=1 min_rank=7 imbalance_ratio=1.853668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33097, 32763, 32771, 31460, 32768, 33094, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31460 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39944 max_load_reduction_pct=0.546871 moved_to_replicas=75684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] avg=32768.0000 max=33098 max_rank=6 min=31490 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=27643 max_load_reduction_pct=0.455096 moved_to_replicas=38272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[60741, 26314, 32765, 33108, 32720, 32784, 43711, 1] avg=32768.0000 max=60741 max_rank=0 min=1 min_rank=7 imbalance_ratio=1.853668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[60741, 26314, 32765, 33108, 32720, 32784, 43711, 1] avg=32768.0000 max=60741 max_rank=0 min=1 min_rank=7 imbalance_ratio=1.853668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] avg=32768.0000 max=33098 max_rank=6 min=31490 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=27643 max_load_reduction_pct=0.455096 moved_to_replicas=38272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[60741, 26314, 32765, 33108, 32720, 32784, 43711, 1] avg=32768.0000 max=60741 max_rank=0 min=1 min_rank=7 imbalance_ratio=1.853668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[60741, 26314, 32765, 33108, 32720, 32784, 43711, 1] avg=32768.0000 max=60741 max_rank=0 min=1 min_rank=7 imbalance_ratio=1.853668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] avg=32768.0000 max=33098 max_rank=6 min=31490 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=27643 max_load_reduction_pct=0.455096 moved_to_replicas=38272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] avg=32768.0000 max=33098 max_rank=6 min=31490 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=27643 max_load_reduction_pct=0.455096 moved_to_replicas=38272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] avg=32768.0000 max=33098 max_rank=6 min=31490 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=27643 max_load_reduction_pct=0.455096 moved_to_replicas=38272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[33097, 33094, 32765, 33096, 32720, 32784, 33098, 31490] avg=32768.0000 max=33098 max_rank=6 min=31490 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=27643 max_load_reduction_pct=0.455096 moved_to_replicas=38272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=10 total_expected_recv=326225 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79616 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=10 total_expected_recv=328336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77952 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=10 total_expected_recv=323805 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=82304 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=10 total_expected_recv=327321 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78464 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=10 total_expected_recv=329953 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=10 total_expected_recv=327800 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78848 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=10 total_expected_recv=328033 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78848 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=10 total_expected_recv=329967 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75520 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:52:01] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 248.1 | throughput per GPU (TFLOP/s/GPU): 497.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.699395E+00 | loss scale: 1.0 | grad norm: 4.446 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=31642 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32192 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=31642 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31104 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=30848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=30848 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31344 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31344 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32224 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32224 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=31083 expected_overflow=0 exact_recv_per_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=31083 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32027 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=31869 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32027 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=31869 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=31034 expected_overflow=0 exact_recv_per_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=31034 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=31749 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=31819 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32096 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=31749 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=31819 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32096 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32121 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32121 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=31448 expected_overflow=0 exact_recv_per_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=31448 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[37416, 35690, 32192, 29636, 33637, 26041, 31101, 36431] avg=32768.0000 max=37416 max_rank=0 min=26041 min_rank=5 imbalance_ratio=1.141846 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[37416, 35690, 32192, 29636, 33637, 26041, 31101, 36431] avg=32768.0000 max=37416 max_rank=0 min=26041 min_rank=5 imbalance_ratio=1.141846 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] avg=32768.0000 max=33097 max_rank=1 min=31642 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4319 max_load_reduction_pct=0.115432 moved_to_replicas=10790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[37416, 35690, 32192, 29636, 33637, 26041, 31101, 36431] avg=32768.0000 max=37416 max_rank=0 min=26041 min_rank=5 imbalance_ratio=1.141846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[37416, 35690, 32192, 29636, 33637, 26041, 31101, 36431] avg=32768.0000 max=37416 max_rank=0 min=26041 min_rank=5 imbalance_ratio=1.141846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] avg=32768.0000 max=33097 max_rank=1 min=31642 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4319 max_load_reduction_pct=0.115432 moved_to_replicas=10790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[37416, 35690, 32192, 29636, 33637, 26041, 31101, 36431] avg=32768.0000 max=37416 max_rank=0 min=26041 min_rank=5 imbalance_ratio=1.141846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[37416, 35690, 32192, 29636, 33637, 26041, 31101, 36431] avg=32768.0000 max=37416 max_rank=0 min=26041 min_rank=5 imbalance_ratio=1.141846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] avg=32768.0000 max=33097 max_rank=1 min=31642 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4319 max_load_reduction_pct=0.115432 moved_to_replicas=10790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] avg=32768.0000 max=33097 max_rank=1 min=31642 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4319 max_load_reduction_pct=0.115432 moved_to_replicas=10790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[37416, 35690, 32192, 29636, 33637, 26041, 31101, 36431] avg=32768.0000 max=37416 max_rank=0 min=26041 min_rank=5 imbalance_ratio=1.141846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[29045, 56326, 17745, 11106, 18408, 17781, 54991, 56742] avg=32768.0000 max=56742 max_rank=7 min=11106 min_rank=3 imbalance_ratio=1.731628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[37416, 35690, 32192, 29636, 33637, 26041, 31101, 36431] avg=32768.0000 max=37416 max_rank=0 min=26041 min_rank=5 imbalance_ratio=1.141846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] avg=32768.0000 max=33097 max_rank=1 min=31642 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4319 max_load_reduction_pct=0.115432 moved_to_replicas=10790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] avg=32768.0000 max=33097 max_rank=1 min=31642 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4319 max_load_reduction_pct=0.115432 moved_to_replicas=10790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[29045, 56326, 17745, 11106, 18408, 17781, 54991, 56742] avg=32768.0000 max=56742 max_rank=7 min=11106 min_rank=3 imbalance_ratio=1.731628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=23645 max_load_reduction_pct=0.416711 moved_to_replicas=68771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] avg=32768.0000 max=33097 max_rank=1 min=31642 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4319 max_load_reduction_pct=0.115432 moved_to_replicas=10790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[33096, 33097, 32192, 32971, 33096, 32954, 31642, 33096] avg=32768.0000 max=33097 max_rank=1 min=31642 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4319 max_load_reduction_pct=0.115432 moved_to_replicas=10790 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[29045, 56326, 17745, 11106, 18408, 17781, 54991, 56742] avg=32768.0000 max=56742 max_rank=7 min=11106 min_rank=3 imbalance_ratio=1.731628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[29045, 56326, 17745, 11106, 18408, 17781, 54991, 56742] avg=32768.0000 max=56742 max_rank=7 min=11106 min_rank=3 imbalance_ratio=1.731628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=23645 max_load_reduction_pct=0.416711 moved_to_replicas=68771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[29045, 56326, 17745, 11106, 18408, 17781, 54991, 56742] avg=32768.0000 max=56742 max_rank=7 min=11106 min_rank=3 imbalance_ratio=1.731628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=23645 max_load_reduction_pct=0.416711 moved_to_replicas=68771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=23645 max_load_reduction_pct=0.416711 moved_to_replicas=68771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[29045, 56326, 17745, 11106, 18408, 17781, 54991, 56742] avg=32768.0000 max=56742 max_rank=7 min=11106 min_rank=3 imbalance_ratio=1.731628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[27976, 17482, 57142, 18922, 54191, 28032, 36, 58363] avg=32768.0000 max=58363 max_rank=7 min=36 min_rank=6 imbalance_ratio=1.781097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[29045, 56326, 17745, 11106, 18408, 17781, 54991, 56742] avg=32768.0000 max=56742 max_rank=7 min=11106 min_rank=3 imbalance_ratio=1.731628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[29045, 56326, 17745, 11106, 18408, 17781, 54991, 56742] avg=32768.0000 max=56742 max_rank=7 min=11106 min_rank=3 imbalance_ratio=1.731628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=23645 max_load_reduction_pct=0.416711 moved_to_replicas=68771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[27976, 17482, 57142, 18922, 54191, 28032, 36, 58363] avg=32768.0000 max=58363 max_rank=7 min=36 min_rank=6 imbalance_ratio=1.781097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=23645 max_load_reduction_pct=0.416711 moved_to_replicas=68771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25266 max_load_reduction_pct=0.432911 moved_to_replicas=70408 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=23645 max_load_reduction_pct=0.416711 moved_to_replicas=68771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[27976, 17482, 57142, 18922, 54191, 28032, 36, 58363] avg=32768.0000 max=58363 max_rank=7 min=36 min_rank=6 imbalance_ratio=1.781097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[27976, 17482, 57142, 18922, 54191, 28032, 36, 58363] avg=32768.0000 max=58363 max_rank=7 min=36 min_rank=6 imbalance_ratio=1.781097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=23645 max_load_reduction_pct=0.416711 moved_to_replicas=68771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25266 max_load_reduction_pct=0.432911 moved_to_replicas=70408 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[27976, 17482, 57142, 18922, 54191, 28032, 36, 58363] avg=32768.0000 max=58363 max_rank=7 min=36 min_rank=6 imbalance_ratio=1.781097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25266 max_load_reduction_pct=0.432911 moved_to_replicas=70408 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25266 max_load_reduction_pct=0.432911 moved_to_replicas=70408 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[45749, 9623, 11577, 36421, 20300, 22543, 32224, 83707] avg=32768.0000 max=83707 max_rank=7 min=9623 min_rank=1 imbalance_ratio=2.554535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[27976, 17482, 57142, 18922, 54191, 28032, 36, 58363] avg=32768.0000 max=58363 max_rank=7 min=36 min_rank=6 imbalance_ratio=1.781097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[27976, 17482, 57142, 18922, 54191, 28032, 36, 58363] avg=32768.0000 max=58363 max_rank=7 min=36 min_rank=6 imbalance_ratio=1.781097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[27976, 17482, 57142, 18922, 54191, 28032, 36, 58363] avg=32768.0000 max=58363 max_rank=7 min=36 min_rank=6 imbalance_ratio=1.781097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25266 max_load_reduction_pct=0.432911 moved_to_replicas=70408 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[45749, 9623, 11577, 36421, 20300, 22543, 32224, 83707] avg=32768.0000 max=83707 max_rank=7 min=9623 min_rank=1 imbalance_ratio=2.554535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] avg=32768.0000 max=33096 max_rank=0 min=31344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50611 max_load_reduction_pct=0.604621 moved_to_replicas=66589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25266 max_load_reduction_pct=0.432911 moved_to_replicas=70408 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25266 max_load_reduction_pct=0.432911 moved_to_replicas=70408 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[45749, 9623, 11577, 36421, 20300, 22543, 32224, 83707] avg=32768.0000 max=83707 max_rank=7 min=9623 min_rank=1 imbalance_ratio=2.554535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[45749, 9623, 11577, 36421, 20300, 22543, 32224, 83707] avg=32768.0000 max=83707 max_rank=7 min=9623 min_rank=1 imbalance_ratio=2.554535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 30472, 33094, 33097] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25266 max_load_reduction_pct=0.432911 moved_to_replicas=70408 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] avg=32768.0000 max=33096 max_rank=0 min=31344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50611 max_load_reduction_pct=0.604621 moved_to_replicas=66589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[45749, 9623, 11577, 36421, 20300, 22543, 32224, 83707] avg=32768.0000 max=83707 max_rank=7 min=9623 min_rank=1 imbalance_ratio=2.554535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[8374, 111, 15, 9328, 101403, 58525, 65758, 18630] avg=32768.0000 max=101403 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.094574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[45749, 9623, 11577, 36421, 20300, 22543, 32224, 83707] avg=32768.0000 max=83707 max_rank=7 min=9623 min_rank=1 imbalance_ratio=2.554535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] avg=32768.0000 max=33096 max_rank=0 min=31344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50611 max_load_reduction_pct=0.604621 moved_to_replicas=66589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] avg=32768.0000 max=33096 max_rank=0 min=31344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50611 max_load_reduction_pct=0.604621 moved_to_replicas=66589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[45749, 9623, 11577, 36421, 20300, 22543, 32224, 83707] avg=32768.0000 max=83707 max_rank=7 min=9623 min_rank=1 imbalance_ratio=2.554535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[45749, 9623, 11577, 36421, 20300, 22543, 32224, 83707] avg=32768.0000 max=83707 max_rank=7 min=9623 min_rank=1 imbalance_ratio=2.554535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] avg=32768.0000 max=33096 max_rank=0 min=31344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50611 max_load_reduction_pct=0.604621 moved_to_replicas=66589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31083 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68307 max_load_reduction_pct=0.673619 moved_to_replicas=126398 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[8374, 111, 15, 9328, 101403, 58525, 65758, 18630] avg=32768.0000 max=101403 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.094574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] avg=32768.0000 max=33096 max_rank=0 min=31344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50611 max_load_reduction_pct=0.604621 moved_to_replicas=66589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] avg=32768.0000 max=33096 max_rank=0 min=31344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50611 max_load_reduction_pct=0.604621 moved_to_replicas=66589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[8374, 111, 15, 9328, 101403, 58525, 65758, 18630] avg=32768.0000 max=101403 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.094574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[8374, 111, 15, 9328, 101403, 58525, 65758, 18630] avg=32768.0000 max=101403 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.094574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[33096, 33096, 31344, 33096, 33096, 33096, 32224, 33096] avg=32768.0000 max=33096 max_rank=0 min=31344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50611 max_load_reduction_pct=0.604621 moved_to_replicas=66589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31083 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68307 max_load_reduction_pct=0.673619 moved_to_replicas=126398 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[8374, 111, 15, 9328, 101403, 58525, 65758, 18630] avg=32768.0000 max=101403 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.094574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31083 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68307 max_load_reduction_pct=0.673619 moved_to_replicas=126398 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[98069, 19414, 2371, 31869, 3, 21658, 70028, 18732] avg=32768.0000 max=98069 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.992828 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31083 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68307 max_load_reduction_pct=0.673619 moved_to_replicas=126398 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[8374, 111, 15, 9328, 101403, 58525, 65758, 18630] avg=32768.0000 max=101403 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.094574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[8374, 111, 15, 9328, 101403, 58525, 65758, 18630] avg=32768.0000 max=101403 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.094574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[8374, 111, 15, 9328, 101403, 58525, 65758, 18630] avg=32768.0000 max=101403 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.094574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31083 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68307 max_load_reduction_pct=0.673619 moved_to_replicas=126398 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31869 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64972 max_load_reduction_pct=0.662513 moved_to_replicas=101905 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[98069, 19414, 2371, 31869, 3, 21658, 70028, 18732] avg=32768.0000 max=98069 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.992828 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31083 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68307 max_load_reduction_pct=0.673619 moved_to_replicas=126398 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31083 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68307 max_load_reduction_pct=0.673619 moved_to_replicas=126398 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[98069, 19414, 2371, 31869, 3, 21658, 70028, 18732] avg=32768.0000 max=98069 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.992828 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[98069, 19414, 2371, 31869, 3, 21658, 70028, 18732] avg=32768.0000 max=98069 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.992828 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[31083, 32799, 32782, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31083 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68307 max_load_reduction_pct=0.673619 moved_to_replicas=126398 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31869 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64972 max_load_reduction_pct=0.662513 moved_to_replicas=101905 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[98069, 19414, 2371, 31869, 3, 21658, 70028, 18732] avg=32768.0000 max=98069 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.992828 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31869 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64972 max_load_reduction_pct=0.662513 moved_to_replicas=101905 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31869 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64972 max_load_reduction_pct=0.662513 moved_to_replicas=101905 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[30138, 30983, 13435, 32851, 12, 45443, 91743, 17539] avg=32768.0000 max=91743 max_rank=6 min=12 min_rank=4 imbalance_ratio=2.799774 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[98069, 19414, 2371, 31869, 3, 21658, 70028, 18732] avg=32768.0000 max=98069 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.992828 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[98069, 19414, 2371, 31869, 3, 21658, 70028, 18732] avg=32768.0000 max=98069 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.992828 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[98069, 19414, 2371, 31869, 3, 21658, 70028, 18732] avg=32768.0000 max=98069 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.992828 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31869 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64972 max_load_reduction_pct=0.662513 moved_to_replicas=101905 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[30138, 30983, 13435, 32851, 12, 45443, 91743, 17539] avg=32768.0000 max=91743 max_rank=6 min=12 min_rank=4 imbalance_ratio=2.799774 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31034 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=58646 max_load_reduction_pct=0.639242 moved_to_replicas=70994 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31869 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64972 max_load_reduction_pct=0.662513 moved_to_replicas=101905 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31869 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64972 max_load_reduction_pct=0.662513 moved_to_replicas=101905 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[30138, 30983, 13435, 32851, 12, 45443, 91743, 17539] avg=32768.0000 max=91743 max_rank=6 min=12 min_rank=4 imbalance_ratio=2.799774 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[30138, 30983, 13435, 32851, 12, 45443, 91743, 17539] avg=32768.0000 max=91743 max_rank=6 min=12 min_rank=4 imbalance_ratio=2.799774 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 32769, 32027, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31869 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64972 max_load_reduction_pct=0.662513 moved_to_replicas=101905 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31034 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=58646 max_load_reduction_pct=0.639242 moved_to_replicas=70994 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[30138, 30983, 13435, 32851, 12, 45443, 91743, 17539] avg=32768.0000 max=91743 max_rank=6 min=12 min_rank=4 imbalance_ratio=2.799774 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31034 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=58646 max_load_reduction_pct=0.639242 moved_to_replicas=70994 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[3120, 42807, 5359, 87677, 42426, 49775, 29847, 1133] avg=32768.0000 max=87677 max_rank=3 min=1133 min_rank=7 imbalance_ratio=2.675690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31034 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=58646 max_load_reduction_pct=0.639242 moved_to_replicas=70994 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[30138, 30983, 13435, 32851, 12, 45443, 91743, 17539] avg=32768.0000 max=91743 max_rank=6 min=12 min_rank=4 imbalance_ratio=2.799774 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[30138, 30983, 13435, 32851, 12, 45443, 91743, 17539] avg=32768.0000 max=91743 max_rank=6 min=12 min_rank=4 imbalance_ratio=2.799774 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31034 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=58646 max_load_reduction_pct=0.639242 moved_to_replicas=70994 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[30138, 30983, 13435, 32851, 12, 45443, 91743, 17539] avg=32768.0000 max=91743 max_rank=6 min=12 min_rank=4 imbalance_ratio=2.799774 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[3120, 42807, 5359, 87677, 42426, 49775, 29847, 1133] avg=32768.0000 max=87677 max_rank=3 min=1133 min_rank=7 imbalance_ratio=2.675690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] avg=32768.0000 max=33097 max_rank=4 min=31749 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=54580 max_load_reduction_pct=0.622512 moved_to_replicas=90301 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31034 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=58646 max_load_reduction_pct=0.639242 moved_to_replicas=70994 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31034 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=58646 max_load_reduction_pct=0.639242 moved_to_replicas=70994 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[3120, 42807, 5359, 87677, 42426, 49775, 29847, 1133] avg=32768.0000 max=87677 max_rank=3 min=1133 min_rank=7 imbalance_ratio=2.675690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[3120, 42807, 5359, 87677, 42426, 49775, 29847, 1133] avg=32768.0000 max=87677 max_rank=3 min=1133 min_rank=7 imbalance_ratio=2.675690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[33096, 31034, 33097, 32851, 32779, 33094, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31034 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=58646 max_load_reduction_pct=0.639242 moved_to_replicas=70994 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] avg=32768.0000 max=33097 max_rank=4 min=31749 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=54580 max_load_reduction_pct=0.622512 moved_to_replicas=90301 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[3120, 42807, 5359, 87677, 42426, 49775, 29847, 1133] avg=32768.0000 max=87677 max_rank=3 min=1133 min_rank=7 imbalance_ratio=2.675690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] avg=32768.0000 max=33097 max_rank=4 min=31749 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=54580 max_load_reduction_pct=0.622512 moved_to_replicas=90301 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] avg=32768.0000 max=33097 max_rank=4 min=31749 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=54580 max_load_reduction_pct=0.622512 moved_to_replicas=90301 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[760, 32774, 14071, 32096, 1, 98295, 56552, 27595] avg=32768.0000 max=98295 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[3120, 42807, 5359, 87677, 42426, 49775, 29847, 1133] avg=32768.0000 max=87677 max_rank=3 min=1133 min_rank=7 imbalance_ratio=2.675690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[3120, 42807, 5359, 87677, 42426, 49775, 29847, 1133] avg=32768.0000 max=87677 max_rank=3 min=1133 min_rank=7 imbalance_ratio=2.675690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] avg=32768.0000 max=33097 max_rank=4 min=31749 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=54580 max_load_reduction_pct=0.622512 moved_to_replicas=90301 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[3120, 42807, 5359, 87677, 42426, 49775, 29847, 1133] avg=32768.0000 max=87677 max_rank=3 min=1133 min_rank=7 imbalance_ratio=2.675690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] avg=32768.0000 max=33097 max_rank=6 min=32096 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65198 max_load_reduction_pct=0.663289 moved_to_replicas=88655 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[760, 32774, 14071, 32096, 1, 98295, 56552, 27595] avg=32768.0000 max=98295 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] avg=32768.0000 max=33097 max_rank=4 min=31749 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=54580 max_load_reduction_pct=0.622512 moved_to_replicas=90301 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[760, 32774, 14071, 32096, 1, 98295, 56552, 27595] avg=32768.0000 max=98295 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[760, 32774, 14071, 32096, 1, 98295, 56552, 27595] avg=32768.0000 max=98295 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] avg=32768.0000 max=33097 max_rank=4 min=31749 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=54580 max_load_reduction_pct=0.622512 moved_to_replicas=90301 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33095, 33095, 31749, 33096, 33097, 33097, 31819, 33096] avg=32768.0000 max=33097 max_rank=4 min=31749 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=54580 max_load_reduction_pct=0.622512 moved_to_replicas=90301 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] avg=32768.0000 max=33097 max_rank=6 min=32096 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65198 max_load_reduction_pct=0.663289 moved_to_replicas=88655 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[760, 32774, 14071, 32096, 1, 98295, 56552, 27595] avg=32768.0000 max=98295 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] avg=32768.0000 max=33097 max_rank=6 min=32096 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65198 max_load_reduction_pct=0.663289 moved_to_replicas=88655 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] avg=32768.0000 max=33097 max_rank=6 min=32096 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65198 max_load_reduction_pct=0.663289 moved_to_replicas=88655 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[39236, 12, 32778, 58183, 64536, 32729, 34662, 8] avg=32768.0000 max=64536 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.969482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[760, 32774, 14071, 32096, 1, 98295, 56552, 27595] avg=32768.0000 max=98295 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[760, 32774, 14071, 32096, 1, 98295, 56552, 27595] avg=32768.0000 max=98295 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] avg=32768.0000 max=33097 max_rank=6 min=32096 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65198 max_load_reduction_pct=0.663289 moved_to_replicas=88655 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[760, 32774, 14071, 32096, 1, 98295, 56552, 27595] avg=32768.0000 max=98295 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] avg=32768.0000 max=33097 max_rank=0 min=31448 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31439 max_load_reduction_pct=0.487154 moved_to_replicas=64233 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[39236, 12, 32778, 58183, 64536, 32729, 34662, 8] avg=32768.0000 max=64536 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.969482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[39236, 12, 32778, 58183, 64536, 32729, 34662, 8] avg=32768.0000 max=64536 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.969482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] avg=32768.0000 max=33097 max_rank=6 min=32096 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65198 max_load_reduction_pct=0.663289 moved_to_replicas=88655 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] avg=32768.0000 max=33097 max_rank=6 min=32096 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65198 max_load_reduction_pct=0.663289 moved_to_replicas=88655 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[39236, 12, 32778, 58183, 64536, 32729, 34662, 8] avg=32768.0000 max=64536 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.969482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[33096, 32774, 33096, 32096, 32768, 33096, 33097, 32121] avg=32768.0000 max=33097 max_rank=6 min=32096 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65198 max_load_reduction_pct=0.663289 moved_to_replicas=88655 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] avg=32768.0000 max=33097 max_rank=0 min=31448 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31439 max_load_reduction_pct=0.487154 moved_to_replicas=64233 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] avg=32768.0000 max=33097 max_rank=0 min=31448 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31439 max_load_reduction_pct=0.487154 moved_to_replicas=64233 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[39236, 12, 32778, 58183, 64536, 32729, 34662, 8] avg=32768.0000 max=64536 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.969482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] avg=32768.0000 max=33097 max_rank=0 min=31448 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31439 max_load_reduction_pct=0.487154 moved_to_replicas=64233 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[39236, 12, 32778, 58183, 64536, 32729, 34662, 8] avg=32768.0000 max=64536 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.969482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[39236, 12, 32778, 58183, 64536, 32729, 34662, 8] avg=32768.0000 max=64536 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.969482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] avg=32768.0000 max=33097 max_rank=0 min=31448 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31439 max_load_reduction_pct=0.487154 moved_to_replicas=64233 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[39236, 12, 32778, 58183, 64536, 32729, 34662, 8] avg=32768.0000 max=64536 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.969482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] avg=32768.0000 max=33097 max_rank=0 min=31448 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31439 max_load_reduction_pct=0.487154 moved_to_replicas=64233 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] avg=32768.0000 max=33097 max_rank=0 min=31448 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31439 max_load_reduction_pct=0.487154 moved_to_replicas=64233 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32807, 32778, 33093, 33096, 32729, 33096, 31448] avg=32768.0000 max=33097 max_rank=0 min=31448 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31439 max_load_reduction_pct=0.487154 moved_to_replicas=64233 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=10 total_expected_recv=328360 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78336 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=10 total_expected_recv=326756 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79360 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=10 total_expected_recv=328338 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78080 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=10 total_expected_recv=329991 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=10 total_expected_recv=326323 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79616 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=10 total_expected_recv=326326 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=80128 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=10 total_expected_recv=327357 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79616 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=10 total_expected_recv=327989 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79488 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:52:01] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 239.3 | throughput per GPU (TFLOP/s/GPU): 516.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.638415E+00 | loss scale: 1.0 | grad norm: 4.366 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32038 expected_overflow=0 exact_recv_per_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32016 expected_overflow=0 exact_recv_per_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32038 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32016 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=31809 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=31809 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32282 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=30976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=30976 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=30679 expected_overflow=0 exact_recv_per_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=30679 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=31987 expected_overflow=0 exact_recv_per_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=31880 expected_overflow=0 exact_recv_per_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=31987 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=31880 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=31825 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=31825 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32225 expected_overflow=0 exact_recv_per_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32225 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32338 expected_overflow=0 exact_recv_per_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32135 expected_overflow=0 exact_recv_per_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33062 expected_overflow=0 exact_recv_per_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32338 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32135 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33062 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=31869 expected_overflow=0 exact_recv_per_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=31869 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=31840 expected_overflow=0 exact_recv_per_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=31840 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=31225 expected_overflow=0 exact_recv_per_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[37212, 32612, 27393, 29329, 35804, 28153, 35951, 35690] avg=32768.0000 max=37212 max_rank=0 min=27393 min_rank=2 imbalance_ratio=1.135620 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=31225 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[37212, 32612, 27393, 29329, 35804, 28153, 35951, 35690] avg=32768.0000 max=37212 max_rank=0 min=27393 min_rank=2 imbalance_ratio=1.135620 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[37212, 32612, 27393, 29329, 35804, 28153, 35951, 35690] avg=32768.0000 max=37212 max_rank=0 min=27393 min_rank=2 imbalance_ratio=1.135620 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4116 max_load_reduction_pct=0.110609 moved_to_replicas=12273 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[37212, 32612, 27393, 29329, 35804, 28153, 35951, 35690] avg=32768.0000 max=37212 max_rank=0 min=27393 min_rank=2 imbalance_ratio=1.135620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4116 max_load_reduction_pct=0.110609 moved_to_replicas=12273 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[37212, 32612, 27393, 29329, 35804, 28153, 35951, 35690] avg=32768.0000 max=37212 max_rank=0 min=27393 min_rank=2 imbalance_ratio=1.135620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[37212, 32612, 27393, 29329, 35804, 28153, 35951, 35690] avg=32768.0000 max=37212 max_rank=0 min=27393 min_rank=2 imbalance_ratio=1.135620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4116 max_load_reduction_pct=0.110609 moved_to_replicas=12273 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4116 max_load_reduction_pct=0.110609 moved_to_replicas=12273 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[37212, 32612, 27393, 29329, 35804, 28153, 35951, 35690] avg=32768.0000 max=37212 max_rank=0 min=27393 min_rank=2 imbalance_ratio=1.135620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[6233, 53102, 9319, 5313, 9003, 22832, 56350, 99992] avg=32768.0000 max=99992 max_rank=7 min=5313 min_rank=3 imbalance_ratio=3.051514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4116 max_load_reduction_pct=0.110609 moved_to_replicas=12273 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[37212, 32612, 27393, 29329, 35804, 28153, 35951, 35690] avg=32768.0000 max=37212 max_rank=0 min=27393 min_rank=2 imbalance_ratio=1.135620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4116 max_load_reduction_pct=0.110609 moved_to_replicas=12273 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[6233, 53102, 9319, 5313, 9003, 22832, 56350, 99992] avg=32768.0000 max=99992 max_rank=7 min=5313 min_rank=3 imbalance_ratio=3.051514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[6233, 53102, 9319, 5313, 9003, 22832, 56350, 99992] avg=32768.0000 max=99992 max_rank=7 min=5313 min_rank=3 imbalance_ratio=3.051514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31809 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=66895 max_load_reduction_pct=0.669004 moved_to_replicas=110156 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4116 max_load_reduction_pct=0.110609 moved_to_replicas=12273 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[6233, 53102, 9319, 5313, 9003, 22832, 56350, 99992] avg=32768.0000 max=99992 max_rank=7 min=5313 min_rank=3 imbalance_ratio=3.051514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[33095, 32612, 32016, 32038, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4116 max_load_reduction_pct=0.110609 moved_to_replicas=12273 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31809 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=66895 max_load_reduction_pct=0.669004 moved_to_replicas=110156 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[6233, 53102, 9319, 5313, 9003, 22832, 56350, 99992] avg=32768.0000 max=99992 max_rank=7 min=5313 min_rank=3 imbalance_ratio=3.051514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[6233, 53102, 9319, 5313, 9003, 22832, 56350, 99992] avg=32768.0000 max=99992 max_rank=7 min=5313 min_rank=3 imbalance_ratio=3.051514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31809 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=66895 max_load_reduction_pct=0.669004 moved_to_replicas=110156 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31809 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=66895 max_load_reduction_pct=0.669004 moved_to_replicas=110156 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[30136, 2187, 40709, 2198, 82331, 36997, 4200, 63386] avg=32768.0000 max=82331 max_rank=4 min=2187 min_rank=1 imbalance_ratio=2.512543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[6233, 53102, 9319, 5313, 9003, 22832, 56350, 99992] avg=32768.0000 max=99992 max_rank=7 min=5313 min_rank=3 imbalance_ratio=3.051514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[30136, 2187, 40709, 2198, 82331, 36997, 4200, 63386] avg=32768.0000 max=82331 max_rank=4 min=2187 min_rank=1 imbalance_ratio=2.512543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31809 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=66895 max_load_reduction_pct=0.669004 moved_to_replicas=110156 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31809 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=66895 max_load_reduction_pct=0.669004 moved_to_replicas=110156 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[6233, 53102, 9319, 5313, 9003, 22832, 56350, 99992] avg=32768.0000 max=99992 max_rank=7 min=5313 min_rank=3 imbalance_ratio=3.051514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[30136, 2187, 40709, 2198, 82331, 36997, 4200, 63386] avg=32768.0000 max=82331 max_rank=4 min=2187 min_rank=1 imbalance_ratio=2.512543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=49234 max_load_reduction_pct=0.598001 moved_to_replicas=91039 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31809 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=66895 max_load_reduction_pct=0.669004 moved_to_replicas=110156 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[30136, 2187, 40709, 2198, 82331, 36997, 4200, 63386] avg=32768.0000 max=82331 max_rank=4 min=2187 min_rank=1 imbalance_ratio=2.512543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=49234 max_load_reduction_pct=0.598001 moved_to_replicas=91039 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32282, 33096, 32573, 31809, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31809 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=66895 max_load_reduction_pct=0.669004 moved_to_replicas=110156 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=49234 max_load_reduction_pct=0.598001 moved_to_replicas=91039 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[30136, 2187, 40709, 2198, 82331, 36997, 4200, 63386] avg=32768.0000 max=82331 max_rank=4 min=2187 min_rank=1 imbalance_ratio=2.512543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[30136, 2187, 40709, 2198, 82331, 36997, 4200, 63386] avg=32768.0000 max=82331 max_rank=4 min=2187 min_rank=1 imbalance_ratio=2.512543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[36777, 5345, 78, 33369, 62330, 33656, 57596, 32993] avg=32768.0000 max=62330 max_rank=4 min=78 min_rank=2 imbalance_ratio=1.902161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=49234 max_load_reduction_pct=0.598001 moved_to_replicas=91039 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[30136, 2187, 40709, 2198, 82331, 36997, 4200, 63386] avg=32768.0000 max=82331 max_rank=4 min=2187 min_rank=1 imbalance_ratio=2.512543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[36777, 5345, 78, 33369, 62330, 33656, 57596, 32993] avg=32768.0000 max=62330 max_rank=4 min=78 min_rank=2 imbalance_ratio=1.902161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[30136, 2187, 40709, 2198, 82331, 36997, 4200, 63386] avg=32768.0000 max=82331 max_rank=4 min=2187 min_rank=1 imbalance_ratio=2.512543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=49234 max_load_reduction_pct=0.598001 moved_to_replicas=91039 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=49234 max_load_reduction_pct=0.598001 moved_to_replicas=91039 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] avg=32768.0000 max=33096 max_rank=0 min=30679 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29234 max_load_reduction_pct=0.469020 moved_to_replicas=58248 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[36777, 5345, 78, 33369, 62330, 33656, 57596, 32993] avg=32768.0000 max=62330 max_rank=4 min=78 min_rank=2 imbalance_ratio=1.902161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=49234 max_load_reduction_pct=0.598001 moved_to_replicas=91039 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] avg=32768.0000 max=33096 max_rank=0 min=30679 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29234 max_load_reduction_pct=0.469020 moved_to_replicas=58248 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[36777, 5345, 78, 33369, 62330, 33656, 57596, 32993] avg=32768.0000 max=62330 max_rank=4 min=78 min_rank=2 imbalance_ratio=1.902161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=49234 max_load_reduction_pct=0.598001 moved_to_replicas=91039 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] avg=32768.0000 max=33096 max_rank=0 min=30679 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29234 max_load_reduction_pct=0.469020 moved_to_replicas=58248 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[36777, 5345, 78, 33369, 62330, 33656, 57596, 32993] avg=32768.0000 max=62330 max_rank=4 min=78 min_rank=2 imbalance_ratio=1.902161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[36777, 5345, 78, 33369, 62330, 33656, 57596, 32993] avg=32768.0000 max=62330 max_rank=4 min=78 min_rank=2 imbalance_ratio=1.902161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[25699, 34, 4533, 3080, 65580, 27362, 98136, 37720] avg=32768.0000 max=98136 max_rank=6 min=34 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] avg=32768.0000 max=33096 max_rank=0 min=30679 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29234 max_load_reduction_pct=0.469020 moved_to_replicas=58248 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[36777, 5345, 78, 33369, 62330, 33656, 57596, 32993] avg=32768.0000 max=62330 max_rank=4 min=78 min_rank=2 imbalance_ratio=1.902161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[25699, 34, 4533, 3080, 65580, 27362, 98136, 37720] avg=32768.0000 max=98136 max_rank=6 min=34 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[36777, 5345, 78, 33369, 62330, 33656, 57596, 32993] avg=32768.0000 max=62330 max_rank=4 min=78 min_rank=2 imbalance_ratio=1.902161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] avg=32768.0000 max=33096 max_rank=0 min=30679 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29234 max_load_reduction_pct=0.469020 moved_to_replicas=58248 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] avg=32768.0000 max=33098 max_rank=2 min=31880 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65038 max_load_reduction_pct=0.662733 moved_to_replicas=102148 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] avg=32768.0000 max=33096 max_rank=0 min=30679 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29234 max_load_reduction_pct=0.469020 moved_to_replicas=58248 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[25699, 34, 4533, 3080, 65580, 27362, 98136, 37720] avg=32768.0000 max=98136 max_rank=6 min=34 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] avg=32768.0000 max=33096 max_rank=0 min=30679 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29234 max_load_reduction_pct=0.469020 moved_to_replicas=58248 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] avg=32768.0000 max=33098 max_rank=2 min=31880 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65038 max_load_reduction_pct=0.662733 moved_to_replicas=102148 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[25699, 34, 4533, 3080, 65580, 27362, 98136, 37720] avg=32768.0000 max=98136 max_rank=6 min=34 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[33096, 30679, 32993, 33096, 33096, 33096, 33095, 32993] avg=32768.0000 max=33096 max_rank=0 min=30679 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29234 max_load_reduction_pct=0.469020 moved_to_replicas=58248 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] avg=32768.0000 max=33098 max_rank=2 min=31880 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65038 max_load_reduction_pct=0.662733 moved_to_replicas=102148 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[25699, 34, 4533, 3080, 65580, 27362, 98136, 37720] avg=32768.0000 max=98136 max_rank=6 min=34 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[98223, 345, 32820, 1233, 12, 31275, 64590, 33646] avg=32768.0000 max=98223 max_rank=0 min=12 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[25699, 34, 4533, 3080, 65580, 27362, 98136, 37720] avg=32768.0000 max=98136 max_rank=6 min=34 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[98223, 345, 32820, 1233, 12, 31275, 64590, 33646] avg=32768.0000 max=98223 max_rank=0 min=12 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] avg=32768.0000 max=33098 max_rank=2 min=31880 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65038 max_load_reduction_pct=0.662733 moved_to_replicas=102148 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[25699, 34, 4533, 3080, 65580, 27362, 98136, 37720] avg=32768.0000 max=98136 max_rank=6 min=34 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[25699, 34, 4533, 3080, 65580, 27362, 98136, 37720] avg=32768.0000 max=98136 max_rank=6 min=34 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] avg=32768.0000 max=33098 max_rank=2 min=31880 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65038 max_load_reduction_pct=0.662733 moved_to_replicas=102148 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[98223, 345, 32820, 1233, 12, 31275, 64590, 33646] avg=32768.0000 max=98223 max_rank=0 min=12 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31825 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97171 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] avg=32768.0000 max=33098 max_rank=2 min=31880 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65038 max_load_reduction_pct=0.662733 moved_to_replicas=102148 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31825 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97171 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] avg=32768.0000 max=33098 max_rank=2 min=31880 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65038 max_load_reduction_pct=0.662733 moved_to_replicas=102148 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[31880, 32796, 33098, 33096, 33096, 31987, 33096, 33095] avg=32768.0000 max=33098 max_rank=2 min=31880 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65038 max_load_reduction_pct=0.662733 moved_to_replicas=102148 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31825 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97171 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[98223, 345, 32820, 1233, 12, 31275, 64590, 33646] avg=32768.0000 max=98223 max_rank=0 min=12 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[98223, 345, 32820, 1233, 12, 31275, 64590, 33646] avg=32768.0000 max=98223 max_rank=0 min=12 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32135, 28701, 32338, 65145, 19610, 35329, 48710, 176] avg=32768.0000 max=65145 max_rank=3 min=176 min_rank=7 imbalance_ratio=1.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[98223, 345, 32820, 1233, 12, 31275, 64590, 33646] avg=32768.0000 max=98223 max_rank=0 min=12 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32135, 28701, 32338, 65145, 19610, 35329, 48710, 176] avg=32768.0000 max=65145 max_rank=3 min=176 min_rank=7 imbalance_ratio=1.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[98223, 345, 32820, 1233, 12, 31275, 64590, 33646] avg=32768.0000 max=98223 max_rank=0 min=12 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31825 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97171 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32135, 28701, 32338, 65145, 19610, 35329, 48710, 176] avg=32768.0000 max=65145 max_rank=3 min=176 min_rank=7 imbalance_ratio=1.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[98223, 345, 32820, 1233, 12, 31275, 64590, 33646] avg=32768.0000 max=98223 max_rank=0 min=12 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31825 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97171 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] avg=32768.0000 max=33097 max_rank=4 min=32135 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32048 max_load_reduction_pct=0.491949 moved_to_replicas=49896 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31825 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97171 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] avg=32768.0000 max=33097 max_rank=4 min=32135 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32048 max_load_reduction_pct=0.491949 moved_to_replicas=49896 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31825 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97171 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] avg=32768.0000 max=33097 max_rank=4 min=32135 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32048 max_load_reduction_pct=0.491949 moved_to_replicas=49896 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32712, 32820, 32727, 32772, 31825, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31825 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97171 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32135, 28701, 32338, 65145, 19610, 35329, 48710, 176] avg=32768.0000 max=65145 max_rank=3 min=176 min_rank=7 imbalance_ratio=1.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32135, 28701, 32338, 65145, 19610, 35329, 48710, 176] avg=32768.0000 max=65145 max_rank=3 min=176 min_rank=7 imbalance_ratio=1.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[108, 3806, 31283, 65593, 65362, 32753, 34346, 28893] avg=32768.0000 max=65593 max_rank=3 min=108 min_rank=0 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32135, 28701, 32338, 65145, 19610, 35329, 48710, 176] avg=32768.0000 max=65145 max_rank=3 min=176 min_rank=7 imbalance_ratio=1.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[108, 3806, 31283, 65593, 65362, 32753, 34346, 28893] avg=32768.0000 max=65593 max_rank=3 min=108 min_rank=0 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32135, 28701, 32338, 65145, 19610, 35329, 48710, 176] avg=32768.0000 max=65145 max_rank=3 min=176 min_rank=7 imbalance_ratio=1.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] avg=32768.0000 max=33097 max_rank=4 min=32135 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32048 max_load_reduction_pct=0.491949 moved_to_replicas=49896 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[108, 3806, 31283, 65593, 65362, 32753, 34346, 28893] avg=32768.0000 max=65593 max_rank=3 min=108 min_rank=0 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] avg=32768.0000 max=33097 max_rank=4 min=32135 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32048 max_load_reduction_pct=0.491949 moved_to_replicas=49896 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32135, 28701, 32338, 65145, 19610, 35329, 48710, 176] avg=32768.0000 max=65145 max_rank=3 min=176 min_rank=7 imbalance_ratio=1.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] avg=32768.0000 max=33096 max_rank=1 min=31869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32497 max_load_reduction_pct=0.495434 moved_to_replicas=66013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] avg=32768.0000 max=33097 max_rank=4 min=32135 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32048 max_load_reduction_pct=0.491949 moved_to_replicas=49896 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] avg=32768.0000 max=33096 max_rank=1 min=31869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32497 max_load_reduction_pct=0.495434 moved_to_replicas=66013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] avg=32768.0000 max=33097 max_rank=4 min=32135 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32048 max_load_reduction_pct=0.491949 moved_to_replicas=49896 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] avg=32768.0000 max=33096 max_rank=1 min=31869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32497 max_load_reduction_pct=0.495434 moved_to_replicas=66013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[108, 3806, 31283, 65593, 65362, 32753, 34346, 28893] avg=32768.0000 max=65593 max_rank=3 min=108 min_rank=0 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32135, 33062, 32338, 33096, 33097, 33096, 33095, 32225] avg=32768.0000 max=33097 max_rank=4 min=32135 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32048 max_load_reduction_pct=0.491949 moved_to_replicas=49896 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[108, 3806, 31283, 65593, 65362, 32753, 34346, 28893] avg=32768.0000 max=65593 max_rank=3 min=108 min_rank=0 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32755, 27180, 20, 31840, 709, 98260, 38598, 32782] avg=32768.0000 max=98260 max_rank=5 min=20 min_rank=2 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[108, 3806, 31283, 65593, 65362, 32753, 34346, 28893] avg=32768.0000 max=65593 max_rank=3 min=108 min_rank=0 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32755, 27180, 20, 31840, 709, 98260, 38598, 32782] avg=32768.0000 max=98260 max_rank=5 min=20 min_rank=2 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[108, 3806, 31283, 65593, 65362, 32753, 34346, 28893] avg=32768.0000 max=65593 max_rank=3 min=108 min_rank=0 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] avg=32768.0000 max=33096 max_rank=1 min=31869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32497 max_load_reduction_pct=0.495434 moved_to_replicas=66013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32755, 27180, 20, 31840, 709, 98260, 38598, 32782] avg=32768.0000 max=98260 max_rank=5 min=20 min_rank=2 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] avg=32768.0000 max=33096 max_rank=1 min=31869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32497 max_load_reduction_pct=0.495434 moved_to_replicas=66013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[108, 3806, 31283, 65593, 65362, 32753, 34346, 28893] avg=32768.0000 max=65593 max_rank=3 min=108 min_rank=0 imbalance_ratio=2.001740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] avg=32768.0000 max=33096 max_rank=4 min=31840 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=70666 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] avg=32768.0000 max=33096 max_rank=1 min=31869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32497 max_load_reduction_pct=0.495434 moved_to_replicas=66013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] avg=32768.0000 max=33096 max_rank=4 min=31840 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=70666 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] avg=32768.0000 max=33096 max_rank=1 min=31869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32497 max_load_reduction_pct=0.495434 moved_to_replicas=66013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] avg=32768.0000 max=33096 max_rank=4 min=31840 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=70666 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32755, 27180, 20, 31840, 709, 98260, 38598, 32782] avg=32768.0000 max=98260 max_rank=5 min=20 min_rank=2 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32605, 33096, 32533, 33096, 33096, 32753, 33096, 31869] avg=32768.0000 max=33096 max_rank=1 min=31869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32497 max_load_reduction_pct=0.495434 moved_to_replicas=66013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49804, 1005, 32946, 61486, 46348, 81, 65509, 4965] avg=32768.0000 max=65509 max_rank=6 min=81 min_rank=5 imbalance_ratio=1.999176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32755, 27180, 20, 31840, 709, 98260, 38598, 32782] avg=32768.0000 max=98260 max_rank=5 min=20 min_rank=2 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32755, 27180, 20, 31840, 709, 98260, 38598, 32782] avg=32768.0000 max=98260 max_rank=5 min=20 min_rank=2 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49804, 1005, 32946, 61486, 46348, 81, 65509, 4965] avg=32768.0000 max=65509 max_rank=6 min=81 min_rank=5 imbalance_ratio=1.999176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32755, 27180, 20, 31840, 709, 98260, 38598, 32782] avg=32768.0000 max=98260 max_rank=5 min=20 min_rank=2 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] avg=32768.0000 max=33096 max_rank=4 min=31840 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=70666 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49804, 1005, 32946, 61486, 46348, 81, 65509, 4965] avg=32768.0000 max=65509 max_rank=6 min=81 min_rank=5 imbalance_ratio=1.999176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32755, 27180, 20, 31840, 709, 98260, 38598, 32782] avg=32768.0000 max=98260 max_rank=5 min=20 min_rank=2 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31225 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32412 max_load_reduction_pct=0.494772 moved_to_replicas=90763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] avg=32768.0000 max=33096 max_rank=4 min=31840 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=70666 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] avg=32768.0000 max=33096 max_rank=4 min=31840 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=70666 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31225 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32412 max_load_reduction_pct=0.494772 moved_to_replicas=90763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] avg=32768.0000 max=33096 max_rank=4 min=31840 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=70666 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31225 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32412 max_load_reduction_pct=0.494772 moved_to_replicas=90763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49804, 1005, 32946, 61486, 46348, 81, 65509, 4965] avg=32768.0000 max=65509 max_rank=6 min=81 min_rank=5 imbalance_ratio=1.999176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32755, 32697, 32782, 31840, 33096, 33096, 33096, 32782] avg=32768.0000 max=33096 max_rank=4 min=31840 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=70666 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[49804, 1005, 32946, 61486, 46348, 81, 65509, 4965] avg=32768.0000 max=65509 max_rank=6 min=81 min_rank=5 imbalance_ratio=1.999176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49804, 1005, 32946, 61486, 46348, 81, 65509, 4965] avg=32768.0000 max=65509 max_rank=6 min=81 min_rank=5 imbalance_ratio=1.999176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49804, 1005, 32946, 61486, 46348, 81, 65509, 4965] avg=32768.0000 max=65509 max_rank=6 min=81 min_rank=5 imbalance_ratio=1.999176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31225 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32412 max_load_reduction_pct=0.494772 moved_to_replicas=90763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49804, 1005, 32946, 61486, 46348, 81, 65509, 4965] avg=32768.0000 max=65509 max_rank=6 min=81 min_rank=5 imbalance_ratio=1.999176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31225 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32412 max_load_reduction_pct=0.494772 moved_to_replicas=90763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31225 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32412 max_load_reduction_pct=0.494772 moved_to_replicas=90763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31225 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32412 max_load_reduction_pct=0.494772 moved_to_replicas=90763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31225, 32946, 33095, 33097, 32494, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31225 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32412 max_load_reduction_pct=0.494772 moved_to_replicas=90763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=10 total_expected_recv=328444 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75776 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=10 total_expected_recv=326988 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77568 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=10 total_expected_recv=330637 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=73856 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=10 total_expected_recv=327636 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=10 total_expected_recv=324511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=80640 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=10 total_expected_recv=327195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=10 total_expected_recv=325071 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78976 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=10 total_expected_recv=330958 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=74240 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:52:01] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 234.1 | throughput per GPU (TFLOP/s/GPU): 527.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.562642E+00 | loss scale: 1.0 | grad norm: 4.308 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +W0826 15:52:01.775000 146553 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +[after training is done] datetime: 2026-08-26 15:52:01 +W0826 15:52:01.776000 146553 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 146643 closing signal SIGTERM +W0826 15:52:01.777000 146553 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 146644 closing signal SIGTERM +W0826 15:52:01.782000 146553 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 146645 closing signal SIGTERM +W0826 15:52:01.787000 146553 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 146646 closing signal SIGTERM +W0826 15:52:01.790000 146553 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 146647 closing signal SIGTERM +W0826 15:52:01.794000 146553 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 146648 closing signal SIGTERM +W0826 15:52:01.800000 146553 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 146649 closing signal SIGTERM +W0826 15:52:01.807000 146553 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 146650 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -1864,21 +6076,17 @@ Traceback (most recent call last): File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ return launch_agent(self._config, self._entrypoint, list(args)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -pretrain_gpt.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2026-08-26_15:05:14 - host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 60312) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 146553 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4d5f56b54d901670696f6928ff3e7fd68e2c28d --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,64 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..bde626fe3fc16cf3a1ef6e23eff565a5f1368367 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..43074f6ef06acb0f263ddd377ad9d43ad43998a2 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,647 @@ +[15:52:16] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + QWEN2_57B / ultraep (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:52:17.546000 149908 torch/distributed/run.py:766] +W0826 15:52:17.546000 149908 torch/distributed/run.py:766] ***************************************** +W0826 15:52:17.546000 149908 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:52:17.546000 149908 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank3]:[W826 15:52:36.064778057 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:52:36.064811881 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:52:36.064811429 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:52:36.064844464 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:52:36.064870774 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:52:36.064952671 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:52:36.065191833 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:52:36.219780669 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.961 seconds +time to initialize megatron (seconds): 6.921 +[after megatron is initialized] datetime: 2026-08-26 15:52:42 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv index a511e023cab4401e71f166d970318fad75cb2cba..68346c427fbe1917a737695f5154092e6d790bf4 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 4 1, 1 2, 1 3, 1 @@ -25,7 +25,7 @@ 0, 4 1, 4 2, 4 -3, 1 +3, 4 4, 1 5, 1 6, 1 @@ -44,88 +44,56 @@ 3, 4 4, 4 5, 4 -6, 4 +6, 47 7, 4 -0, 4 -1, 4 -2, 4 -3, 629 -4, 4 -5, 629 -6, 629 -7, 4 -0, 725 -1, 725 -2, 725 -3, 725 -4, 725 -5, 725 -6, 725 -7, 725 -0, 725 -1, 725 -2, 725 -3, 725 -4, 725 -5, 725 -6, 725 -7, 725 0, 821 -1, 819 -2, 819 +1, 821 +2, 821 3, 821 -4, 819 +4, 821 5, 821 -6, 819 -7, 819 -0, 1537 -1, 1537 -2, 1537 -3, 1537 -4, 1537 -5, 1537 -6, 1537 -7, 1537 -0, 2551 -1, 2551 -2, 2551 -3, 2551 -4, 2551 -5, 2551 -6, 2551 -7, 2551 -0, 2959 -1, 2959 -2, 2959 -3, 2959 -4, 2959 -5, 2959 -6, 2959 -7, 2959 -0, 51537 -1, 51537 -2, 51537 -3, 51537 -4, 51249 -5, 51537 -6, 51537 -7, 51537 -0, 52789 -1, 52789 -2, 52789 -3, 52789 -4, 52501 -5, 52789 -6, 52791 -7, 52789 -0, 53651 -1, 53631 -2, 53647 -3, 53643 -4, 53349 -5, 53629 -6, 53633 -7, 53625 +6, 821 +7, 821 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 2417 +1, 2419 +2, 2421 +3, 2403 +4, 2407 +5, 2407 +6, 2431 +7, 2437 +0, 22733 +1, 15429 +2, 15431 +3, 15431 +4, 15427 +5, 15427 +6, 29645 +7, 15033 +0, 51917 +1, 51895 +2, 51937 +3, 51919 +4, 51703 +5, 51925 +6, 51917 +7, 51935 +0, 53635 +1, 53601 +2, 53605 +3, 53581 +4, 53301 +5, 53585 +6, 53595 +7, 53583 0, 54649 1, 54649 2, 54649 @@ -150,22 +118,22 @@ 5, 57201 6, 57201 7, 57201 -0, 57201 -1, 57201 -2, 57201 -3, 57201 -4, 56913 -5, 57201 -6, 57201 -7, 57201 -0, 90609 -1, 90671 -2, 90681 -3, 90681 -4, 90613 -5, 90667 -6, 90665 -7, 90647 +0, 60853 +1, 60853 +2, 60853 +3, 60853 +4, 60201 +5, 60853 +6, 60853 +7, 60853 +0, 90695 +1, 90743 +2, 90743 +3, 90743 +4, 90727 +5, 90743 +6, 90743 +7, 90743 0, 91083 1, 91131 2, 91131 @@ -182,30 +150,30 @@ 5, 91131 6, 91131 7, 91131 -0, 91083 -1, 91131 -2, 91131 -3, 91131 -4, 91125 -5, 91131 -6, 91131 -7, 91131 -0, 91375 -1, 91441 -2, 91463 -3, 91479 -4, 91453 -5, 91427 -6, 91417 -7, 91429 -0, 95881 -1, 95959 -2, 95935 -3, 95965 -4, 96031 -5, 95921 -6, 95991 -7, 95941 +0, 91331 +1, 91465 +2, 91397 +3, 91475 +4, 91355 +5, 91395 +6, 91389 +7, 91397 +0, 95905 +1, 96031 +2, 95969 +3, 96045 +4, 96083 +5, 95949 +6, 95975 +7, 95961 +0, 109105 +1, 109153 +2, 111233 +3, 109153 +4, 109265 +5, 109153 +6, 116783 +7, 109153 0, 122617 1, 122751 2, 122751 @@ -222,19 +190,35 @@ 5, 122751 6, 122751 7, 122751 -0, 122617 -1, 122751 -2, 122751 -3, 122751 -4, 122655 -5, 122751 -6, 122751 -7, 122751 -0, 5277 -1, 3678 -2, 3678 -3, 3678 -4, 3678 -5, 3678 -6, 3998 -7, 122749 +0, 118287 +1, 118421 +2, 118421 +3, 118421 +4, 118325 +5, 118421 +6, 118421 +7, 118421 +0, 116217 +1, 116741 +2, 116635 +3, 116741 +4, 116257 +5, 116741 +6, 116353 +7, 116511 +0, 4 +1, 114673 +2, 114673 +3, 115061 +4, 114577 +5, 114673 +6, 114673 +7, 114673 +0, 1 +1, 4 +2, 4 +3, 114671 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log index 5d22cd504e81df888c4a8196c2e98835d8153b60..3462b1a209f83b4939c9c27e43b9c0363716611b 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/train.log @@ -1,5 +1,5 @@ -[14:58:13] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[14:58:13] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[15:43:42] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:43:42] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) ============================================================== QWEN3_235B / ultraep (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 @@ -8,14 +8,15 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-UltraEP @ eaf829e7 ============================================================== -W0826 14:58:14.393000 44602 torch/distributed/run.py:766] -W0826 14:58:14.393000 44602 torch/distributed/run.py:766] ***************************************** -W0826 14:58:14.393000 44602 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 14:58:14.393000 44602 torch/distributed/run.py:766] ***************************************** +W0826 15:43:43.695000 128245 torch/distributed/run.py:766] +W0826 15:43:43.695000 128245 torch/distributed/run.py:766] ***************************************** +W0826 15:43:43.695000 128245 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:43:43.695000 128245 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -570,77 +571,64 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... - > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS -> initializing torch distributed ... /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3[Gloo] Rank is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 77 - is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank is connected to 172 is connected to peer ranks. is connected to 7Expected number of connected peer ranks is : 7 peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 + peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank 0 is connected to 0 is connected to 0 is connected to 0 peer ranks. 0 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : 0 +0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank5]:[W826 14:58:32.833865692 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 14:58:32.833935134 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 14:58:32.833955635 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 14:58:32.834270641 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 14:58:32.834296419 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 14:58:32.838303487 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 14:58:32.838344680 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:44:02.153872724 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:44:02.153895536 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:44:02.153922430 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:44:02.153922778 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:44:02.153991626 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:44:02.154047341 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:44:02.156142430 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' -g++ -O3 -Wall -shared -std=c++11 -fPIC -fdiagnostics-color -I/usr/include/python3.12 -I/usr/local/lib/python3.12/dist-packages/pybind11/include helpers.cpp -o helpers_cpp.cpython-312-x86_64-linux-gnu.so -In file included from helpers.cpp:12: -In function ‘pybind11::ssize_t pybind11::detail::byte_offset_unsafe(const Strides&, pybind11::ssize_t, Ix ...) [with long int Dim = 0; Strides = std::array; Ix = {}]’, - inlined from ‘const T& pybind11::detail::unchecked_reference::operator()(Ix ...) const [with Ix = {long int}; T = long int; long int Dims = 1]’ at /usr/local/lib/python3.12/dist-packages/pybind11/include/pybind11/numpy.h:545:65, - inlined from ‘const T& pybind11::detail::unchecked_reference::operator[](pybind11::ssize_t) const [with long int D = 1; = void; T = long int; long int Dims = 1]’ at /usr/local/lib/python3.12/dist-packages/pybind11/include/pybind11/numpy.h:553:26, - inlined from ‘void build_exhaustive_blending_indices(pybind11::array_t&, pybind11::array_t&, const pybind11::array_t&, int32_t)’ at helpers.cpp:67:31: -/usr/local/lib/python3.12/dist-packages/pybind11/include/pybind11/numpy.h:494:14: warning: ‘error_argmax’ may be used uninitialized [-Wmaybe-uninitialized] - 494 | return i * strides[Dim] + byte_offset_unsafe(strides, index...); - | ~~^~~~~~~~~~ -helpers.cpp: In function ‘void build_exhaustive_blending_indices(pybind11::array_t&, pybind11::array_t&, const pybind11::array_t&, int32_t)’: -helpers.cpp:49:13: note: ‘error_argmax’ was declared here - 49 | int64_t error_argmax; - | ^~~~~~~~~~~~ +make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 5.101 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 14:58:38.068928693 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.714 seconds -time to initialize megatron (seconds): 12.124 -[after megatron is initialized] datetime: 2026-08-26 14:58:43 +[rank0]:[W826 15:44:02.301717287 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.812 seconds +time to initialize megatron (seconds): 7.377 +[after megatron is initialized] datetime: 2026-08-26 15:44:08 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -684,30 +672,30 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.final_layernorm.weight - module.decoder.layers.11.self_attention.k_layernorm.weight - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.self_attention.q_layernorm.weight Params for bucket 3 (71835904 elements, 71835904 padded size): module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight Params for bucket 4 (71835904 elements, 71835904 padded size): - module.decoder.layers.9.self_attention.q_layernorm.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.k_layernorm.weight module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight Params for bucket 5 (71835904 elements, 71835904 padded size): module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight @@ -721,109 +709,109 @@ Params for bucket 6 (71835904 elements, 71835904 padded size): module.decoder.layers.7.self_attention.linear_qkv.weight module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.7.self_attention.q_layernorm.weight - module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight Params for bucket 7 (71835904 elements, 71835904 padded size): - module.decoder.layers.6.self_attention.q_layernorm.weight - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.k_layernorm.weight module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight Params for bucket 8 (71835904 elements, 71835904 padded size): - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight Params for bucket 9 (71835904 elements, 71835904 padded size): module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.q_layernorm.weight module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.k_layernorm.weight - module.decoder.layers.4.self_attention.q_layernorm.weight Params for bucket 10 (71835904 elements, 71835904 padded size): - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.self_attention.k_layernorm.weight - module.decoder.layers.3.self_attention.q_layernorm.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight Params for bucket 11 (71835904 elements, 71835904 padded size): - module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight Params for bucket 12 (71835904 elements, 71835904 padded size): module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.self_attention.linear_qkv.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight Params for bucket 13 (71835904 elements, 71835904 padded size): - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.k_layernorm.weight module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight Params for bucket 14 (206045184 elements, 206045184 padded size): module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 Params for bucket 1 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight14 - module.decoder.layers.11.mlp.experts.linear_fc2.weight12 - module.decoder.layers.11.mlp.experts.linear_fc2.weight10 module.decoder.layers.11.mlp.experts.linear_fc2.weight15 - module.decoder.layers.11.mlp.experts.linear_fc2.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 Params for bucket 2 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight8 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight13 module.decoder.layers.11.mlp.experts.linear_fc1.weight15 - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight12 - module.decoder.layers.11.mlp.experts.linear_fc1.weight10 module.decoder.layers.11.mlp.experts.linear_fc1.weight11 module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight8 - module.decoder.layers.11.mlp.experts.linear_fc1.weight7 module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 - module.decoder.layers.11.mlp.experts.linear_fc1.weight3 Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight12 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight13 module.decoder.layers.10.mlp.experts.linear_fc2.weight11 @@ -836,8 +824,8 @@ Params for bucket 8 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 Params for bucket 9 (50331648 elements, 50331648 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 @@ -859,62 +847,62 @@ Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight4 Params for bucket 13 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight13 - module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 Params for bucket 14 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight11 - module.decoder.layers.9.mlp.experts.linear_fc2.weight9 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 - module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight15 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight14 - module.decoder.layers.9.mlp.experts.linear_fc1.weight11 - module.decoder.layers.9.mlp.experts.linear_fc1.weight12 module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight10 - module.decoder.layers.9.mlp.experts.linear_fc1.weight8 module.decoder.layers.9.mlp.experts.linear_fc1.weight9 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 Params for bucket 19 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight15 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 Params for bucket 20 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight13 - module.decoder.layers.8.mlp.experts.linear_fc2.weight11 - module.decoder.layers.8.mlp.experts.linear_fc2.weight9 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 Params for bucket 21 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight4 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 @@ -922,9 +910,9 @@ Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight15 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 module.decoder.layers.8.mlp.experts.linear_fc1.weight10 module.decoder.layers.8.mlp.experts.linear_fc1.weight11 - module.decoder.layers.8.mlp.experts.linear_fc1.weight9 Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight8 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 @@ -939,75 +927,75 @@ Params for bucket 26 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight12 - module.decoder.layers.7.mlp.experts.linear_fc2.weight11 module.decoder.layers.7.mlp.experts.linear_fc2.weight13 module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 Params for bucket 27 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight10 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 Params for bucket 28 (50331648 elements, 50331648 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight14 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 - module.decoder.layers.7.mlp.experts.linear_fc1.weight10 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight9 - module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 Params for bucket 32 (44040192 elements, 44040192 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 - module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 Params for bucket 33 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc2.weight9 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 Params for bucket 34 (50331648 elements, 50331648 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc1.weight12 module.decoder.layers.6.mlp.experts.linear_fc1.weight13 module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight9 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight2 @@ -1023,17 +1011,17 @@ Params for bucket 39 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc2.weight9 Params for bucket 40 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 Params for bucket 41 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc1.weight15 Params for bucket 42 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight12 @@ -1051,9 +1039,9 @@ Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 Params for bucket 45 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight12 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 @@ -1068,130 +1056,130 @@ Params for bucket 46 (44040192 elements, 44040192 padded size): Params for bucket 47 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight14 Params for bucket 48 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight13 - module.decoder.layers.4.mlp.experts.linear_fc1.weight10 - module.decoder.layers.4.mlp.experts.linear_fc1.weight12 module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 Params for bucket 49 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight8 Params for bucket 50 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 Params for bucket 51 (44040192 elements, 44040192 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 Params for bucket 52 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight9 - module.decoder.layers.3.mlp.experts.linear_fc2.weight11 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 - module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 Params for bucket 53 (50331648 elements, 50331648 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 Params for bucket 54 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight14 - module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight11 Params for bucket 55 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight10 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.3.mlp.experts.linear_fc1.weight8 Params for bucket 56 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight15 Params for bucket 58 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 - module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 - module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 Params for bucket 59 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc1.weight15 Params for bucket 61 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight12 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 - module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight8 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 Params for bucket 64 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 Params for bucket 65 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight10 - module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 66 (50331648 elements, 50331648 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight15 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight14 Params for bucket 67 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight13 - module.decoder.layers.1.mlp.experts.linear_fc1.weight12 module.decoder.layers.1.mlp.experts.linear_fc1.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 @@ -1199,9 +1187,9 @@ Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight8 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 70 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight13 @@ -1209,8 +1197,8 @@ Params for bucket 70 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc2.weight14 Params for bucket 71 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 module.decoder.layers.0.mlp.experts.linear_fc2.weight10 @@ -1218,34 +1206,34 @@ Params for bucket 71 (44040192 elements, 44040192 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight6 Params for bucket 72 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc2.weight4 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight11 Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight10 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight8 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 - module.decoder.layers.0.mlp.experts.linear_fc1.weight3 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 14:58:47 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:44:11 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1253,7 +1241,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1266,14 +1254,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 14:58:47 +[after dataloaders are built] datetime: 2026-08-26 15:44:11 done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3740.09, 3748.14) - train/valid/test-data-iterators-setup ..........: (69.67, 84.77) training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3686.30, 3701.22) + train/valid/test-data-iterators-setup ..........: (72.23, 81.81) Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 14:58:47 +[before the start of training step] datetime: 2026-08-26 15:44:11 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1296,14 +1284,14 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1324,14 +1312,10 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1340,6 +1324,10 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1387,496 +1375,496 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63972 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=81920 expected_recv=64773 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63972 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=81920 expected_recv=64773 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63972 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=81920 expected_recv=62362 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=81920 expected_recv=62362 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63164 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63972 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63164 actual_valid_recv=64512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=81920 expected_recv=64152 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=81920 expected_recv=64152 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=81920 expected_recv=62362 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=81920 expected_recv=64773 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=81920 expected_recv=62362 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=81920 expected_recv=64773 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=81920 expected_recv=62983 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=81920 expected_recv=62647 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=81920 expected_recv=62983 actual_valid_recv=64384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63164 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=81920 expected_recv=62647 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=81920 expected_recv=62647 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=81920 expected_recv=63164 actual_valid_recv=64512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=81920 expected_recv=64487 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65584 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=81920 expected_recv=64487 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=81920 expected_recv=61713 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65584 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=81920 expected_recv=61553 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=81920 expected_recv=61713 actual_valid_recv=62848 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=62848 slack_slots=19072 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=81920 expected_recv=61553 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65423 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=81920 expected_recv=62983 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65423 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=81920 expected_recv=62983 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66029 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65423 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=81920 expected_recv=62767 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66029 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=81920 expected_recv=61468 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=81920 expected_recv=62767 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=81920 expected_recv=61468 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65669 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65669 actual_valid_recv=66816 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=81920 expected_recv=62767 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=81920 expected_recv=62767 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=81920 expected_recv=61553 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=81920 expected_recv=61553 actual_valid_recv=62976 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=81920 expected_recv=62137 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=81920 expected_recv=62137 actual_valid_recv=63232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63232 slack_slots=18688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=81920 expected_recv=64360 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=81920 expected_recv=62939 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=81920 expected_recv=64360 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=81920 expected_recv=62939 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65000 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=81920 expected_recv=62137 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65000 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=81920 expected_recv=62137 actual_valid_recv=63232 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63232 slack_slots=18688 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=81920 expected_recv=64355 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=81920 expected_recv=64355 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=81920 expected_recv=64368 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=81920 expected_recv=64368 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=81920 expected_recv=64355 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=81920 expected_recv=64355 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=81920 expected_recv=61468 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=81920 expected_recv=61468 actual_valid_recv=62976 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65669 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65669 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=81920 expected_recv=64360 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=81920 expected_recv=64360 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=81920 expected_recv=64291 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=81920 expected_recv=64291 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=81920 expected_recv=62782 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=81920 expected_recv=64291 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=81920 expected_recv=62782 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=81920 expected_recv=64291 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=81920 expected_recv=62845 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=81920 expected_recv=62845 actual_valid_recv=64128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=81920 expected_recv=62845 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=81920 expected_recv=62845 actual_valid_recv=64128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=81920 expected_recv=61713 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=81920 expected_recv=61713 actual_valid_recv=62848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=81920 expected_recv=62782 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=62848 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=81920 expected_recv=62782 actual_valid_recv=64000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66029 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=81920 expected_recv=66029 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65000 expected_overflow=0 exact_recv_per_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65000 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[78399, 78996, 73820, 31538, 79480, 28200, 80723, 73132] avg=65536.0000 max=80723 max_rank=6 min=28200 min_rank=5 imbalance_ratio=1.231735 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[66194, 66192, 66192, 63972, 66191, 63164, 66191, 66192] avg=65536.0000 max=66194 max_rank=0 min=63164 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14529 max_load_reduction_pct=0.179986 moved_to_replicas=67398 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[92689, 20032, 12635, 124411, 73662, 103447, 24229, 73183] avg=65536.0000 max=124411 max_rank=3 min=12635 min_rank=2 imbalance_ratio=1.898361 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[66192, 64773, 66192, 66192, 66192, 66193, 62362, 66192] avg=65536.0000 max=66193 max_rank=5 min=62362 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=58218 max_load_reduction_pct=0.467949 moved_to_replicas=136432 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[51843, 19359, 5638, 128479, 76682, 52077, 121206, 69004] avg=65536.0000 max=128479 max_rank=3 min=5638 min_rank=2 imbalance_ratio=1.960434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[64152, 66193, 66192, 66192, 66192, 62983, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62983 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=62286 max_load_reduction_pct=0.484795 moved_to_replicas=130603 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[69926, 31460, 72015, 46673, 114591, 72523, 8040, 109060] avg=65536.0000 max=114591 max_rank=4 min=8040 min_rank=6 imbalance_ratio=1.748520 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[66192, 66192, 66192, 64487, 66193, 66193, 62647, 66192] avg=65536.0000 max=66193 max_rank=4 min=62647 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=48398 max_load_reduction_pct=0.422354 moved_to_replicas=107155 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=12 total_expected_recv=792885 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=175360 peak_demand=66193 required_capacity=1.010025 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[21347, 92854, 38921, 57614, 34114, 105265, 93167, 81006] avg=65536.0000 max=105265 max_rank=5 min=21347 min_rank=0 imbalance_ratio=1.606216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=12 total_expected_recv=788470 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=180992 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66191, 65584, 66191, 61553, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=61553 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=39072 max_load_reduction_pct=0.371178 moved_to_replicas=107524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[83565, 25933, 74953, 30358, 113830, 37506, 105191, 52952] avg=65536.0000 max=113830 max_rank=4 min=25933 min_rank=1 imbalance_ratio=1.736908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 65423, 66192, 66192, 66192, 61713] avg=65536.0000 max=66192 max_rank=0 min=61713 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=47638 max_load_reduction_pct=0.418501 moved_to_replicas=112771 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=12 total_expected_recv=782949 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[25805, 72943, 25973, 103961, 22802, 47630, 148546, 76628] avg=65536.0000 max=148546 max_rank=6 min=22802 min_rank=4 imbalance_ratio=2.266632 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=12 total_expected_recv=793693 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=174976 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[66192, 66192, 66190, 66193, 61468, 65669, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=61468 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=82353 max_load_reduction_pct=0.554394 moved_to_replicas=137310 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=12 total_expected_recv=785029 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183424 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=12 total_expected_recv=781186 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186752 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=12 total_expected_recv=784940 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[55080, 45582, 128351, 52922, 79080, 51090, 46154, 66029] avg=65536.0000 max=128351 max_rank=2 min=45582 min_rank=1 imbalance_ratio=1.958481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 62939, 66193, 64360, 66193, 66029] avg=65536.0000 max=66193 max_rank=4 min=62939 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62158 max_load_reduction_pct=0.484281 moved_to_replicas=75047 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=12 total_expected_recv=782304 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186368 peak_demand=66193 required_capacity=1.010025 - [2026-08-26 14:59:18] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 30537.2 | throughput per GPU (TFLOP/s/GPU): 5.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116001E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67456 | min safe capacity: 1.0293 | -Number of parameters in transformer block in billions: 29.85 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[41545, 105960, 89503, 39703, 92780, 29832, 77761, 47204] avg=65536.0000 max=105960 max_rank=1 min=29832 min_rank=5 imbalance_ratio=1.616821 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=12 total_expected_recv=784940 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=12 total_expected_recv=792885 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=175360 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[62767, 66192, 66192, 64368, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=62767 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=39767 max_load_reduction_pct=0.375302 moved_to_replicas=101236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=12 total_expected_recv=781186 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186752 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[21095, 139086, 73650, 139187, 2442, 52984, 48033, 47811] avg=65536.0000 max=139187 max_rank=3 min=2442 min_rank=4 imbalance_ratio=2.123825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[62137, 66192, 66192, 66192, 66191, 66192, 66192, 65000] avg=65536.0000 max=66192 max_rank=1 min=62137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=72995 max_load_reduction_pct=0.524438 moved_to_replicas=153347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[29773, 50670, 107510, 130472, 49642, 28049, 18915, 109257] avg=65536.0000 max=130472 max_rank=3 min=18915 min_rank=6 imbalance_ratio=1.990845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[64355, 66193, 66192, 66192, 66191, 62782, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=62782 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=64279 max_load_reduction_pct=0.492665 moved_to_replicas=148663 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[36581, 90750, 98395, 52250, 99149, 10833, 64291, 72039] avg=65536.0000 max=99149 max_rank=4 min=10833 min_rank=5 imbalance_ratio=1.512894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[66193, 66192, 66191, 62845, 66192, 66192, 64291, 66192] avg=65536.0000 max=66193 max_rank=0 min=62845 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=32956 max_load_reduction_pct=0.332389 moved_to_replicas=95565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=12 total_expected_recv=788470 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=180992 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 15:44:39] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 27146.0 | throughput per GPU (TFLOP/s/GPU): 6.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116001E+01 | loss scale: 1.0 | grad norm: 44.808 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67456 | min safe capacity: 1.0293 |Number of parameters in transformer block in billions: 29.85 + Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 @@ -1899,5394 +1887,5395 @@ Theoretical memory footprints: weight and optimizer=216428.96 MB /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63889 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63247 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63247 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63889 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63889 actual_valid_recv=65024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63247 actual_valid_recv=64256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63889 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=81920 expected_recv=63247 actual_valid_recv=64256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63648 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63488 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63648 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63488 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63488 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=81920 expected_recv=63488 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62336 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=81920 expected_recv=62987 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=81920 expected_recv=62987 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=81920 expected_recv=64638 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65703 actual_valid_recv=66816 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=81920 expected_recv=64638 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65703 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65258 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65258 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=81920 expected_recv=61875 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65258 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=81920 expected_recv=61875 expected_overflow=0 exact_recv_per_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65258 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=81920 expected_recv=62098 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=81920 expected_recv=61875 actual_valid_recv=63616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65038 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=81920 expected_recv=62098 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65038 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=81920 expected_recv=62098 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65038 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65038 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63447 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63447 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63447 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63690 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63447 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63690 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66016 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63690 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66016 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=81920 expected_recv=63690 actual_valid_recv=65152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=81920 expected_recv=64139 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=81920 expected_recv=64139 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=81920 expected_recv=63173 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=81920 expected_recv=62621 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=81920 expected_recv=63173 actual_valid_recv=64512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=81920 expected_recv=64139 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=81920 expected_recv=62621 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66016 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=81920 expected_recv=64139 actual_valid_recv=65280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66016 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=81920 expected_recv=64514 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=81920 expected_recv=62621 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=81920 expected_recv=64514 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=81920 expected_recv=62621 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=81920 expected_recv=64514 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=81920 expected_recv=64514 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65920 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=81920 expected_recv=61216 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62080 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65920 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=81920 expected_recv=61216 actual_valid_recv=62208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65920 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=62080 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65920 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=62208 slack_slots=19712 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=81920 expected_recv=61216 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=81920 expected_recv=61216 actual_valid_recv=62208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=62208 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=81920 expected_recv=60943 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=81920 expected_recv=60943 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=81920 expected_recv=60943 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=81920 expected_recv=60943 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[82006, 83164, 69865, 28921, 82177, 26789, 78343, 73023] avg=65536.0000 max=83164 max_rank=1 min=26789 min_rank=5 imbalance_ratio=1.268982 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 63889, 66192, 63247, 66191, 66192] avg=65536.0000 max=66193 max_rank=0 min=63247 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=16971 max_load_reduction_pct=0.204067 moved_to_replicas=71426 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[97044, 22610, 13345, 127597, 71918, 96183, 20531, 75060] avg=65536.0000 max=127597 max_rank=3 min=13345 min_rank=2 imbalance_ratio=1.946976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[66192, 63648, 66192, 66191, 66192, 66192, 63488, 66193] avg=65536.0000 max=66193 max_rank=7 min=63488 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=61404 max_load_reduction_pct=0.481234 moved_to_replicas=136842 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[49044, 28562, 2800, 119357, 95737, 47849, 111982, 68957] avg=65536.0000 max=119357 max_rank=3 min=2800 min_rank=2 imbalance_ratio=1.821243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53165 max_load_reduction_pct=0.445428 moved_to_replicas=131265 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[73833, 36961, 53188, 44938, 131999, 93150, 11512, 78707] avg=65536.0000 max=131999 max_rank=4 min=11512 min_rank=6 imbalance_ratio=2.014145 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[66191, 62987, 65703, 66191, 66193, 66193, 64638, 66192] avg=65536.0000 max=66193 max_rank=4 min=62987 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=65806 max_load_reduction_pct=0.498534 moved_to_replicas=112921 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[27533, 66577, 68461, 40056, 43952, 97882, 97613, 82214] avg=65536.0000 max=97882 max_rank=5 min=27533 min_rank=0 imbalance_ratio=1.493561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[61875, 66192, 66193, 66193, 65258, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=61875 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31689 max_load_reduction_pct=0.323747 moved_to_replicas=81787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[79430, 29406, 61726, 32121, 152166, 26297, 106888, 36254] avg=65536.0000 max=152166 max_rank=4 min=26297 min_rank=5 imbalance_ratio=2.321869 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66193, 65038, 62098, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62098 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=85973 max_load_reduction_pct=0.564995 moved_to_replicas=139908 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[29923, 42421, 37511, 120062, 15717, 43351, 127200, 108103] avg=65536.0000 max=127200 max_rank=6 min=15717 min_rank=4 imbalance_ratio=1.940918 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[66192, 63447, 66192, 66192, 63690, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63447 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61008 max_load_reduction_pct=0.479623 moved_to_replicas=156789 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[56327, 30044, 127394, 63173, 82590, 66016, 47163, 51581] avg=65536.0000 max=127394 max_rank=2 min=30044 min_rank=1 imbalance_ratio=1.943878 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 63173, 66192, 66016, 66192, 64139] avg=65536.0000 max=66192 max_rank=0 min=63173 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=61202 max_load_reduction_pct=0.480415 moved_to_replicas=77600 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[57112, 110746, 81790, 29127, 70803, 28462, 103035, 43213] avg=65536.0000 max=110746 max_rank=1 min=28462 min_rank=5 imbalance_ratio=1.689850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[64514, 66193, 66192, 66191, 66193, 62621, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62621 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=44553 max_load_reduction_pct=0.402299 moved_to_replicas=101606 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[44845, 97640, 98804, 103355, 15388, 50036, 42552, 71668] avg=65536.0000 max=103355 max_rank=3 min=15388 min_rank=4 imbalance_ratio=1.577072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[60944, 66192, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=60944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37163 max_load_reduction_pct=0.359567 moved_to_replicas=106699 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[37611, 44601, 148258, 97731, 57705, 28833, 48333, 61216] avg=65536.0000 max=148258 max_rank=2 min=28833 min_rank=5 imbalance_ratio=2.262238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[66193, 65920, 66191, 66192, 66192, 66192, 66192, 61216] avg=65536.0000 max=66193 max_rank=0 min=61216 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=82065 max_load_reduction_pct=0.553528 moved_to_replicas=113605 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[40728, 80464, 109845, 34664, 84263, 53341, 40798, 80185] avg=65536.0000 max=109845 max_rank=2 min=34664 min_rank=3 imbalance_ratio=1.676102 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=12 total_expected_recv=782364 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186880 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=12 total_expected_recv=790871 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=177280 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=12 total_expected_recv=785540 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182912 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=12 total_expected_recv=787276 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182784 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66193, 66192, 60943, 66192] avg=65536.0000 max=66193 max_rank=4 min=60943 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=43652 max_load_reduction_pct=0.397396 moved_to_replicas=89989 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=12 total_expected_recv=792661 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=176896 peak_demand=66193 required_capacity=1.010025 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=12 total_expected_recv=784796 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=66192 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=12 total_expected_recv=784886 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183040 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=12 total_expected_recv=782364 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186880 peak_demand=66193 required_capacity=1.010025 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=12 total_expected_recv=783062 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184064 peak_demand=66193 required_capacity=1.010025 - [2026-08-26 14:59:18] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 496.9 | throughput per GPU (TFLOP/s/GPU): 363.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117607E+01 | loss scale: 1.0 | grad norm: 44.341 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67712 | min safe capacity: 1.0332 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=81920 expected_recv=60945 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=12 total_expected_recv=785540 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182912 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=12 total_expected_recv=787276 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182784 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=12 total_expected_recv=790871 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=177280 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 15:44:39] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 860.7 | throughput per GPU (TFLOP/s/GPU): 210.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117607E+01 | loss scale: 1.0 | grad norm: 44.340 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67712 | min safe capacity: 1.0332 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=61952 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=81920 expected_recv=60945 actual_valid_recv=61824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=61952 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=61824 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=81920 expected_recv=64036 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=81920 expected_recv=64036 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=81920 expected_recv=64021 expected_overflow=0 exact_recv_per_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=81920 expected_recv=64021 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67840 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=81920 expected_recv=63099 expected_overflow=0 exact_recv_per_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=81920 expected_recv=63099 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=81920 expected_recv=62372 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=81920 expected_recv=64764 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=81920 expected_recv=63115 expected_overflow=0 exact_recv_per_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=81920 expected_recv=63115 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=81920 expected_recv=64754 expected_overflow=0 exact_recv_per_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=81920 expected_recv=62372 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=81920 expected_recv=64764 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=81920 expected_recv=64754 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=81920 expected_recv=62382 expected_overflow=0 exact_recv_per_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=81920 expected_recv=62382 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63740 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63740 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63358 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63396 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63396 actual_valid_recv=64640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63358 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63778 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=81920 expected_recv=63778 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=81920 expected_recv=60945 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=81920 expected_recv=60945 actual_valid_recv=62592 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=81920 expected_recv=62784 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=81920 expected_recv=62784 actual_valid_recv=63872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=81920 expected_recv=64542 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65993 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=81920 expected_recv=63744 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65993 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66002 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=81920 expected_recv=64542 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66002 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=81920 expected_recv=63744 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=81920 expected_recv=63591 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=81920 expected_recv=63591 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62080 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=62208 slack_slots=19712 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=62080 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=81920 expected_recv=60945 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=81920 expected_recv=60945 actual_valid_recv=62208 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=62208 slack_slots=19712 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63241 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63895 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=81920 expected_recv=60945 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=81920 expected_recv=60945 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63241 actual_valid_recv=64512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63895 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63715 expected_overflow=0 exact_recv_per_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66057 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63715 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66057 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=81920 expected_recv=63362 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=81920 expected_recv=63362 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63422 expected_overflow=0 exact_recv_per_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=81920 expected_recv=63422 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=81920 expected_recv=63534 expected_overflow=0 exact_recv_per_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=81920 expected_recv=63534 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65741 expected_overflow=0 exact_recv_per_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65746 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=81920 expected_recv=64356 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65746 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=81920 expected_recv=64356 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=81920 expected_recv=63503 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=81920 expected_recv=64776 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=81920 expected_recv=63503 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=81920 expected_recv=64776 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=81920 expected_recv=64553 expected_overflow=0 exact_recv_per_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65741 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=81920 expected_recv=64553 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65692 expected_overflow=0 exact_recv_per_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65692 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=81920 expected_recv=64746 expected_overflow=0 exact_recv_per_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=81920 expected_recv=64746 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65049 expected_overflow=0 exact_recv_per_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62336 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65049 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=81920 expected_recv=63466 expected_overflow=0 exact_recv_per_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=81920 expected_recv=63466 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65116 expected_overflow=0 exact_recv_per_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65116 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[73522, 125324, 92877, 13108, 76291, 17290, 53172, 72704] avg=65536.0000 max=125324 max_rank=1 min=13108 min_rank=3 imbalance_ratio=1.912292 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[73522, 125324, 92877, 13108, 76291, 17290, 53172, 72704] avg=65536.0000 max=125324 max_rank=1 min=13108 min_rank=3 imbalance_ratio=1.912292 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[73554, 125381, 92869, 13097, 76183, 17265, 53183, 72756] avg=65536.0000 max=125381 max_rank=1 min=13097 min_rank=3 imbalance_ratio=1.913162 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[66191, 66192, 66192, 66193, 66191, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=3 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59188 max_load_reduction_pct=0.472065 moved_to_replicas=109783 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[1064, 111779, 879, 50316, 3818, 205373, 55496, 95563] avg=65536.0000 max=205373 max_rank=5 min=879 min_rank=2 imbalance_ratio=3.133743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[63099, 66193, 66192, 66192, 64036, 66192, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63099 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=139180 max_load_reduction_pct=0.677694 moved_to_replicas=214139 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[20652, 131, 200959, 23847, 86859, 118286, 7336, 66218] avg=65536.0000 max=200959 max_rank=2 min=131 min_rank=1 imbalance_ratio=3.066391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[66192, 64764, 66192, 62372, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62372 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=134767 max_load_reduction_pct=0.670619 moved_to_replicas=207554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[59261, 101739, 107211, 70740, 63019, 63740, 17409, 41169] avg=65536.0000 max=107211 max_rank=2 min=17409 min_rank=6 imbalance_ratio=1.635910 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 63396, 63740, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=63396 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=41018 max_load_reduction_pct=0.382591 moved_to_replicas=81114 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[16615, 79687, 130541, 154730, 54707, 5802, 14317, 67889] avg=65536.0000 max=154730 max_rank=3 min=5802 min_rank=5 imbalance_ratio=2.360992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=88538 max_load_reduction_pct=0.572210 moved_to_replicas=168079 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[113859, 79047, 17358, 57895, 173779, 57448, 1041, 23861] avg=65536.0000 max=173779 max_rank=4 min=1041 min_rank=6 imbalance_ratio=2.651657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[66193, 66192, 64542, 66192, 66192, 62784, 66002, 66191] avg=65536.0000 max=66193 max_rank=0 min=62784 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=107586 max_load_reduction_pct=0.619097 moved_to_replicas=168109 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[30498, 21507, 54409, 24995, 235, 99771, 122961, 169912] avg=65536.0000 max=169912 max_rank=7 min=235 min_rank=4 imbalance_ratio=2.592651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[66191, 66191, 66192, 66193, 60944, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103719 max_load_reduction_pct=0.610428 moved_to_replicas=194068 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[254784, 16787, 37029, 70213, 39442, 71258, 1369, 33406] avg=65536.0000 max=254784 max_rank=0 min=1369 min_rank=6 imbalance_ratio=3.887695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188591 max_load_reduction_pct=0.740200 moved_to_replicas=197679 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[81155, 1279, 72672, 115945, 7885, 128808, 113815, 2729] avg=65536.0000 max=128808 max_rank=5 min=1279 min_rank=1 imbalance_ratio=1.965454 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[66193, 63895, 66192, 66192, 63241, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=0 min=63241 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62615 max_load_reduction_pct=0.486111 moved_to_replicas=181435 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[6684, 59111, 64356, 14008, 30480, 106155, 177748, 65746] avg=65536.0000 max=177748 max_rank=6 min=6684 min_rank=0 imbalance_ratio=2.712219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63362, 64356, 66057, 66192, 66192, 66191, 65746] avg=65536.0000 max=66192 max_rank=0 min=63362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111556 max_load_reduction_pct=0.627608 moved_to_replicas=151519 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65049, 458, 124386, 66634, 63061, 21580, 126307, 56813] avg=65536.0000 max=126307 max_rank=6 min=458 min_rank=1 imbalance_ratio=1.927292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65049, 64776, 66192, 66192, 63503, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=5 min=63503 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=60114 max_load_reduction_pct=0.475936 moved_to_replicas=118751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[6069, 98234, 39410, 110271, 67056, 53637, 13850, 135761] avg=65536.0000 max=135761 max_rank=7 min=6069 min_rank=0 imbalance_ratio=2.071548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60944, 66191, 66192] avg=65536.0000 max=66193 max_rank=3 min=60944 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=69568 max_load_reduction_pct=0.512430 moved_to_replicas=146554 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=12 total_expected_recv=773216 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193792 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=12 total_expected_recv=790352 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=179200 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=12 total_expected_recv=786334 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=181888 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=12 total_expected_recv=783197 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=12 total_expected_recv=793855 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=173952 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=12 total_expected_recv=790817 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=177408 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=12 total_expected_recv=783618 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=12 total_expected_recv=790067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=179072 peak_demand=66193 required_capacity=1.010025 - [2026-08-26 14:59:18] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 341.3 | throughput per GPU (TFLOP/s/GPU): 529.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685495E+00 | loss scale: 1.0 | grad norm: 62.685 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=81920 expected_recv=64727 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=81920 expected_recv=62409 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=81920 expected_recv=64727 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[73522, 125324, 92877, 13108, 76291, 17290, 53172, 72704] avg=65536.0000 max=125324 max_rank=1 min=13108 min_rank=3 imbalance_ratio=1.912292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[73522, 125324, 92877, 13108, 76291, 17290, 53172, 72704] avg=65536.0000 max=125324 max_rank=1 min=13108 min_rank=3 imbalance_ratio=1.912292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] avg=65536.0000 max=66194 max_rank=3 min=60944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=59130 max_load_reduction_pct=0.471817 moved_to_replicas=109758 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] avg=65536.0000 max=66194 max_rank=3 min=60944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=59130 max_load_reduction_pct=0.471817 moved_to_replicas=109758 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[73522, 125324, 92877, 13108, 76291, 17290, 53172, 72704] avg=65536.0000 max=125324 max_rank=1 min=13108 min_rank=3 imbalance_ratio=1.912292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] avg=65536.0000 max=66194 max_rank=3 min=60944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=59130 max_load_reduction_pct=0.471817 moved_to_replicas=109758 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[73522, 125324, 92877, 13108, 76291, 17290, 53172, 72704] avg=65536.0000 max=125324 max_rank=1 min=13108 min_rank=3 imbalance_ratio=1.912292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] avg=65536.0000 max=66194 max_rank=3 min=60944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=59130 max_load_reduction_pct=0.471817 moved_to_replicas=109758 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[73522, 125324, 92877, 13108, 76291, 17290, 53172, 72704] avg=65536.0000 max=125324 max_rank=1 min=13108 min_rank=3 imbalance_ratio=1.912292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[73522, 125324, 92877, 13108, 76291, 17290, 53172, 72704] avg=65536.0000 max=125324 max_rank=1 min=13108 min_rank=3 imbalance_ratio=1.912292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] avg=65536.0000 max=66194 max_rank=3 min=60944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=59130 max_load_reduction_pct=0.471817 moved_to_replicas=109758 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[1062, 112052, 855, 50245, 3693, 205345, 55555, 95481] avg=65536.0000 max=205345 max_rank=5 min=855 min_rank=2 imbalance_ratio=3.133316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[1062, 112052, 855, 50245, 3693, 205345, 55555, 95481] avg=65536.0000 max=205345 max_rank=5 min=855 min_rank=2 imbalance_ratio=3.133316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] avg=65536.0000 max=66194 max_rank=3 min=60944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=59130 max_load_reduction_pct=0.471817 moved_to_replicas=109758 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] avg=65536.0000 max=66194 max_rank=3 min=60944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=59130 max_load_reduction_pct=0.471817 moved_to_replicas=109758 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66194, 66191, 66192, 60944, 66191] avg=65536.0000 max=66194 max_rank=3 min=60944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=59130 max_load_reduction_pct=0.471817 moved_to_replicas=109758 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[1062, 112052, 855, 50245, 3693, 205345, 55555, 95481] avg=65536.0000 max=205345 max_rank=5 min=855 min_rank=2 imbalance_ratio=3.133316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63115 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=139153 max_load_reduction_pct=0.677655 moved_to_replicas=214302 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63115 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=139153 max_load_reduction_pct=0.677655 moved_to_replicas=214302 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[1062, 112052, 855, 50245, 3693, 205345, 55555, 95481] avg=65536.0000 max=205345 max_rank=5 min=855 min_rank=2 imbalance_ratio=3.133316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63115 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=139153 max_load_reduction_pct=0.677655 moved_to_replicas=214302 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[1062, 112052, 855, 50245, 3693, 205345, 55555, 95481] avg=65536.0000 max=205345 max_rank=5 min=855 min_rank=2 imbalance_ratio=3.133316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[1062, 112052, 855, 50245, 3693, 205345, 55555, 95481] avg=65536.0000 max=205345 max_rank=5 min=855 min_rank=2 imbalance_ratio=3.133316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[1062, 112052, 855, 50245, 3693, 205345, 55555, 95481] avg=65536.0000 max=205345 max_rank=5 min=855 min_rank=2 imbalance_ratio=3.133316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[1062, 112052, 855, 50245, 3693, 205345, 55555, 95481] avg=65536.0000 max=205345 max_rank=5 min=855 min_rank=2 imbalance_ratio=3.133316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63115 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=139153 max_load_reduction_pct=0.677655 moved_to_replicas=214302 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63115 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=139153 max_load_reduction_pct=0.677655 moved_to_replicas=214302 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[20921, 119, 200986, 23488, 86696, 118458, 7384, 66236] avg=65536.0000 max=200986 max_rank=2 min=119 min_rank=1 imbalance_ratio=3.066803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[20921, 119, 200986, 23488, 86696, 118458, 7384, 66236] avg=65536.0000 max=200986 max_rank=2 min=119 min_rank=1 imbalance_ratio=3.066803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63115 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=139153 max_load_reduction_pct=0.677655 moved_to_replicas=214302 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63115 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=139153 max_load_reduction_pct=0.677655 moved_to_replicas=214302 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[63115, 66192, 66192, 66192, 64021, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63115 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=139153 max_load_reduction_pct=0.677655 moved_to_replicas=214302 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[20921, 119, 200986, 23488, 86696, 118458, 7384, 66236] avg=65536.0000 max=200986 max_rank=2 min=119 min_rank=1 imbalance_ratio=3.066803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62382 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=134793 max_load_reduction_pct=0.670659 moved_to_replicas=207608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62382 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=134793 max_load_reduction_pct=0.670659 moved_to_replicas=207608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[20921, 119, 200986, 23488, 86696, 118458, 7384, 66236] avg=65536.0000 max=200986 max_rank=2 min=119 min_rank=1 imbalance_ratio=3.066803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[20921, 119, 200986, 23488, 86696, 118458, 7384, 66236] avg=65536.0000 max=200986 max_rank=2 min=119 min_rank=1 imbalance_ratio=3.066803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62382 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=134793 max_load_reduction_pct=0.670659 moved_to_replicas=207608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[20921, 119, 200986, 23488, 86696, 118458, 7384, 66236] avg=65536.0000 max=200986 max_rank=2 min=119 min_rank=1 imbalance_ratio=3.066803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[20921, 119, 200986, 23488, 86696, 118458, 7384, 66236] avg=65536.0000 max=200986 max_rank=2 min=119 min_rank=1 imbalance_ratio=3.066803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[20921, 119, 200986, 23488, 86696, 118458, 7384, 66236] avg=65536.0000 max=200986 max_rank=2 min=119 min_rank=1 imbalance_ratio=3.066803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62382 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=134793 max_load_reduction_pct=0.670659 moved_to_replicas=207608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62382 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=134793 max_load_reduction_pct=0.670659 moved_to_replicas=207608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[59287, 101586, 107146, 70695, 63064, 63778, 17392, 41340] avg=65536.0000 max=107146 max_rank=2 min=17392 min_rank=6 imbalance_ratio=1.634918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[59287, 101586, 107146, 70695, 63064, 63778, 17392, 41340] avg=65536.0000 max=107146 max_rank=2 min=17392 min_rank=6 imbalance_ratio=1.634918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62382 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=134793 max_load_reduction_pct=0.670659 moved_to_replicas=207608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62382 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=134793 max_load_reduction_pct=0.670659 moved_to_replicas=207608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[66193, 64754, 66192, 62382, 66192, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62382 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=134793 max_load_reduction_pct=0.670659 moved_to_replicas=207608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[59287, 101586, 107146, 70695, 63064, 63778, 17392, 41340] avg=65536.0000 max=107146 max_rank=2 min=17392 min_rank=6 imbalance_ratio=1.634918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63358 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=40953 max_load_reduction_pct=0.382217 moved_to_replicas=80851 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63358 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=40953 max_load_reduction_pct=0.382217 moved_to_replicas=80851 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[59287, 101586, 107146, 70695, 63064, 63778, 17392, 41340] avg=65536.0000 max=107146 max_rank=2 min=17392 min_rank=6 imbalance_ratio=1.634918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[59287, 101586, 107146, 70695, 63064, 63778, 17392, 41340] avg=65536.0000 max=107146 max_rank=2 min=17392 min_rank=6 imbalance_ratio=1.634918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63358 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=40953 max_load_reduction_pct=0.382217 moved_to_replicas=80851 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[59287, 101586, 107146, 70695, 63064, 63778, 17392, 41340] avg=65536.0000 max=107146 max_rank=2 min=17392 min_rank=6 imbalance_ratio=1.634918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[59287, 101586, 107146, 70695, 63064, 63778, 17392, 41340] avg=65536.0000 max=107146 max_rank=2 min=17392 min_rank=6 imbalance_ratio=1.634918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[59287, 101586, 107146, 70695, 63064, 63778, 17392, 41340] avg=65536.0000 max=107146 max_rank=2 min=17392 min_rank=6 imbalance_ratio=1.634918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[16756, 79771, 130529, 154635, 54820, 5714, 14048, 68015] avg=65536.0000 max=154635 max_rank=3 min=5714 min_rank=5 imbalance_ratio=2.359543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63358 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=40953 max_load_reduction_pct=0.382217 moved_to_replicas=80851 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63358 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=40953 max_load_reduction_pct=0.382217 moved_to_replicas=80851 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[16756, 79771, 130529, 154635, 54820, 5714, 14048, 68015] avg=65536.0000 max=154635 max_rank=3 min=5714 min_rank=5 imbalance_ratio=2.359543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63358 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=40953 max_load_reduction_pct=0.382217 moved_to_replicas=80851 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63358 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=40953 max_load_reduction_pct=0.382217 moved_to_replicas=80851 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[66192, 66193, 66191, 66192, 63358, 63778, 66193, 66191] avg=65536.0000 max=66193 max_rank=1 min=63358 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=40953 max_load_reduction_pct=0.382217 moved_to_replicas=80851 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[16756, 79771, 130529, 154635, 54820, 5714, 14048, 68015] avg=65536.0000 max=154635 max_rank=3 min=5714 min_rank=5 imbalance_ratio=2.359543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=4 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=88442 max_load_reduction_pct=0.571940 moved_to_replicas=168182 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=4 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=88442 max_load_reduction_pct=0.571940 moved_to_replicas=168182 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[16756, 79771, 130529, 154635, 54820, 5714, 14048, 68015] avg=65536.0000 max=154635 max_rank=3 min=5714 min_rank=5 imbalance_ratio=2.359543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[16756, 79771, 130529, 154635, 54820, 5714, 14048, 68015] avg=65536.0000 max=154635 max_rank=3 min=5714 min_rank=5 imbalance_ratio=2.359543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[16756, 79771, 130529, 154635, 54820, 5714, 14048, 68015] avg=65536.0000 max=154635 max_rank=3 min=5714 min_rank=5 imbalance_ratio=2.359543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=4 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=88442 max_load_reduction_pct=0.571940 moved_to_replicas=168182 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[16756, 79771, 130529, 154635, 54820, 5714, 14048, 68015] avg=65536.0000 max=154635 max_rank=3 min=5714 min_rank=5 imbalance_ratio=2.359543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[16756, 79771, 130529, 154635, 54820, 5714, 14048, 68015] avg=65536.0000 max=154635 max_rank=3 min=5714 min_rank=5 imbalance_ratio=2.359543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[114502, 78537, 17332, 57801, 173611, 57695, 1032, 23778] avg=65536.0000 max=173611 max_rank=4 min=1032 min_rank=6 imbalance_ratio=2.649094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=4 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=88442 max_load_reduction_pct=0.571940 moved_to_replicas=168182 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=4 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=88442 max_load_reduction_pct=0.571940 moved_to_replicas=168182 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[114502, 78537, 17332, 57801, 173611, 57695, 1032, 23778] avg=65536.0000 max=173611 max_rank=4 min=1032 min_rank=6 imbalance_ratio=2.649094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=4 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=88442 max_load_reduction_pct=0.571940 moved_to_replicas=168182 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=4 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=88442 max_load_reduction_pct=0.571940 moved_to_replicas=168182 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[66192, 66191, 66192, 66191, 66193, 66192, 60945, 66192] avg=65536.0000 max=66193 max_rank=4 min=60945 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=88442 max_load_reduction_pct=0.571940 moved_to_replicas=168182 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[114502, 78537, 17332, 57801, 173611, 57695, 1032, 23778] avg=65536.0000 max=173611 max_rank=4 min=1032 min_rank=6 imbalance_ratio=2.649094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] avg=65536.0000 max=66194 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=107417 max_load_reduction_pct=0.618722 moved_to_replicas=168074 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] avg=65536.0000 max=66194 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=107417 max_load_reduction_pct=0.618722 moved_to_replicas=168074 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[114502, 78537, 17332, 57801, 173611, 57695, 1032, 23778] avg=65536.0000 max=173611 max_rank=4 min=1032 min_rank=6 imbalance_ratio=2.649094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[114502, 78537, 17332, 57801, 173611, 57695, 1032, 23778] avg=65536.0000 max=173611 max_rank=4 min=1032 min_rank=6 imbalance_ratio=2.649094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] avg=65536.0000 max=66194 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=107417 max_load_reduction_pct=0.618722 moved_to_replicas=168074 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[114502, 78537, 17332, 57801, 173611, 57695, 1032, 23778] avg=65536.0000 max=173611 max_rank=4 min=1032 min_rank=6 imbalance_ratio=2.649094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[114502, 78537, 17332, 57801, 173611, 57695, 1032, 23778] avg=65536.0000 max=173611 max_rank=4 min=1032 min_rank=6 imbalance_ratio=2.649094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[114502, 78537, 17332, 57801, 173611, 57695, 1032, 23778] avg=65536.0000 max=173611 max_rank=4 min=1032 min_rank=6 imbalance_ratio=2.649094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[30720, 21360, 54491, 25028, 237, 99783, 122594, 170075] avg=65536.0000 max=170075 max_rank=7 min=237 min_rank=4 imbalance_ratio=2.595139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] avg=65536.0000 max=66194 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=107417 max_load_reduction_pct=0.618722 moved_to_replicas=168074 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[30720, 21360, 54491, 25028, 237, 99783, 122594, 170075] avg=65536.0000 max=170075 max_rank=7 min=237 min_rank=4 imbalance_ratio=2.595139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] avg=65536.0000 max=66194 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=107417 max_load_reduction_pct=0.618722 moved_to_replicas=168074 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] avg=65536.0000 max=66194 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=107417 max_load_reduction_pct=0.618722 moved_to_replicas=168074 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] avg=65536.0000 max=66194 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=107417 max_load_reduction_pct=0.618722 moved_to_replicas=168074 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[66194, 66192, 63744, 66192, 66192, 63591, 65993, 66190] avg=65536.0000 max=66194 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=107417 max_load_reduction_pct=0.618722 moved_to_replicas=168074 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[30720, 21360, 54491, 25028, 237, 99783, 122594, 170075] avg=65536.0000 max=170075 max_rank=7 min=237 min_rank=4 imbalance_ratio=2.595139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103882 max_load_reduction_pct=0.610801 moved_to_replicas=193876 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103882 max_load_reduction_pct=0.610801 moved_to_replicas=193876 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[30720, 21360, 54491, 25028, 237, 99783, 122594, 170075] avg=65536.0000 max=170075 max_rank=7 min=237 min_rank=4 imbalance_ratio=2.595139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[30720, 21360, 54491, 25028, 237, 99783, 122594, 170075] avg=65536.0000 max=170075 max_rank=7 min=237 min_rank=4 imbalance_ratio=2.595139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103882 max_load_reduction_pct=0.610801 moved_to_replicas=193876 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[30720, 21360, 54491, 25028, 237, 99783, 122594, 170075] avg=65536.0000 max=170075 max_rank=7 min=237 min_rank=4 imbalance_ratio=2.595139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[30720, 21360, 54491, 25028, 237, 99783, 122594, 170075] avg=65536.0000 max=170075 max_rank=7 min=237 min_rank=4 imbalance_ratio=2.595139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[30720, 21360, 54491, 25028, 237, 99783, 122594, 170075] avg=65536.0000 max=170075 max_rank=7 min=237 min_rank=4 imbalance_ratio=2.595139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[254877, 16734, 36878, 70158, 39351, 71390, 1387, 33513] avg=65536.0000 max=254877 max_rank=0 min=1387 min_rank=6 imbalance_ratio=3.889114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[254877, 16734, 36878, 70158, 39351, 71390, 1387, 33513] avg=65536.0000 max=254877 max_rank=0 min=1387 min_rank=6 imbalance_ratio=3.889114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103882 max_load_reduction_pct=0.610801 moved_to_replicas=193876 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103882 max_load_reduction_pct=0.610801 moved_to_replicas=193876 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103882 max_load_reduction_pct=0.610801 moved_to_replicas=193876 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103882 max_load_reduction_pct=0.610801 moved_to_replicas=193876 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188684 max_load_reduction_pct=0.740294 moved_to_replicas=197849 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[66190, 66193, 66192, 66193, 60944, 66193, 66192, 66191] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=103882 max_load_reduction_pct=0.610801 moved_to_replicas=193876 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188684 max_load_reduction_pct=0.740294 moved_to_replicas=197849 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[254877, 16734, 36878, 70158, 39351, 71390, 1387, 33513] avg=65536.0000 max=254877 max_rank=0 min=1387 min_rank=6 imbalance_ratio=3.889114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[254877, 16734, 36878, 70158, 39351, 71390, 1387, 33513] avg=65536.0000 max=254877 max_rank=0 min=1387 min_rank=6 imbalance_ratio=3.889114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[254877, 16734, 36878, 70158, 39351, 71390, 1387, 33513] avg=65536.0000 max=254877 max_rank=0 min=1387 min_rank=6 imbalance_ratio=3.889114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188684 max_load_reduction_pct=0.740294 moved_to_replicas=197849 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[254877, 16734, 36878, 70158, 39351, 71390, 1387, 33513] avg=65536.0000 max=254877 max_rank=0 min=1387 min_rank=6 imbalance_ratio=3.889114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[254877, 16734, 36878, 70158, 39351, 71390, 1387, 33513] avg=65536.0000 max=254877 max_rank=0 min=1387 min_rank=6 imbalance_ratio=3.889114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[81094, 1330, 72735, 116087, 7717, 128577, 114049, 2699] avg=65536.0000 max=128577 max_rank=5 min=1330 min_rank=1 imbalance_ratio=1.961929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[81094, 1330, 72735, 116087, 7717, 128577, 114049, 2699] avg=65536.0000 max=128577 max_rank=5 min=1330 min_rank=1 imbalance_ratio=1.961929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[254877, 16734, 36878, 70158, 39351, 71390, 1387, 33513] avg=65536.0000 max=254877 max_rank=0 min=1387 min_rank=6 imbalance_ratio=3.889114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188684 max_load_reduction_pct=0.740294 moved_to_replicas=197849 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188684 max_load_reduction_pct=0.740294 moved_to_replicas=197849 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188684 max_load_reduction_pct=0.740294 moved_to_replicas=197849 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63422 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62384 max_load_reduction_pct=0.485188 moved_to_replicas=181582 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188684 max_load_reduction_pct=0.740294 moved_to_replicas=197849 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63422 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62384 max_load_reduction_pct=0.485188 moved_to_replicas=181582 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[66191, 66192, 66191, 66192, 60945, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60945 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=188684 max_load_reduction_pct=0.740294 moved_to_replicas=197849 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[81094, 1330, 72735, 116087, 7717, 128577, 114049, 2699] avg=65536.0000 max=128577 max_rank=5 min=1330 min_rank=1 imbalance_ratio=1.961929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[81094, 1330, 72735, 116087, 7717, 128577, 114049, 2699] avg=65536.0000 max=128577 max_rank=5 min=1330 min_rank=1 imbalance_ratio=1.961929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[81094, 1330, 72735, 116087, 7717, 128577, 114049, 2699] avg=65536.0000 max=128577 max_rank=5 min=1330 min_rank=1 imbalance_ratio=1.961929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[81094, 1330, 72735, 116087, 7717, 128577, 114049, 2699] avg=65536.0000 max=128577 max_rank=5 min=1330 min_rank=1 imbalance_ratio=1.961929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63422 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62384 max_load_reduction_pct=0.485188 moved_to_replicas=181582 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[6423, 58972, 64553, 13943, 30797, 106149, 177710, 65741] avg=65536.0000 max=177710 max_rank=6 min=6423 min_rank=0 imbalance_ratio=2.711639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[6423, 58972, 64553, 13943, 30797, 106149, 177710, 65741] avg=65536.0000 max=177710 max_rank=6 min=6423 min_rank=0 imbalance_ratio=2.711639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[81094, 1330, 72735, 116087, 7717, 128577, 114049, 2699] avg=65536.0000 max=128577 max_rank=5 min=1330 min_rank=1 imbalance_ratio=1.961929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63422 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62384 max_load_reduction_pct=0.485188 moved_to_replicas=181582 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63422 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62384 max_load_reduction_pct=0.485188 moved_to_replicas=181582 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[81094, 1330, 72735, 116087, 7717, 128577, 114049, 2699] avg=65536.0000 max=128577 max_rank=5 min=1330 min_rank=1 imbalance_ratio=1.961929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63422 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62384 max_load_reduction_pct=0.485188 moved_to_replicas=181582 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] avg=65536.0000 max=66192 max_rank=0 min=63534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111518 max_load_reduction_pct=0.627528 moved_to_replicas=151475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] avg=65536.0000 max=66192 max_rank=0 min=63534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111518 max_load_reduction_pct=0.627528 moved_to_replicas=151475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63422 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62384 max_load_reduction_pct=0.485188 moved_to_replicas=181582 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[6423, 58972, 64553, 13943, 30797, 106149, 177710, 65741] avg=65536.0000 max=177710 max_rank=6 min=6423 min_rank=0 imbalance_ratio=2.711639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[66192, 63715, 66192, 66191, 63422, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63422 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62384 max_load_reduction_pct=0.485188 moved_to_replicas=181582 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[6423, 58972, 64553, 13943, 30797, 106149, 177710, 65741] avg=65536.0000 max=177710 max_rank=6 min=6423 min_rank=0 imbalance_ratio=2.711639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[6423, 58972, 64553, 13943, 30797, 106149, 177710, 65741] avg=65536.0000 max=177710 max_rank=6 min=6423 min_rank=0 imbalance_ratio=2.711639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[6423, 58972, 64553, 13943, 30797, 106149, 177710, 65741] avg=65536.0000 max=177710 max_rank=6 min=6423 min_rank=0 imbalance_ratio=2.711639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] avg=65536.0000 max=66192 max_rank=0 min=63534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111518 max_load_reduction_pct=0.627528 moved_to_replicas=151475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65116, 494, 124197, 66674, 62984, 21960, 126314, 56549] avg=65536.0000 max=126314 max_rank=6 min=494 min_rank=1 imbalance_ratio=1.927399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65116, 494, 124197, 66674, 62984, 21960, 126314, 56549] avg=65536.0000 max=126314 max_rank=6 min=494 min_rank=1 imbalance_ratio=1.927399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[6423, 58972, 64553, 13943, 30797, 106149, 177710, 65741] avg=65536.0000 max=177710 max_rank=6 min=6423 min_rank=0 imbalance_ratio=2.711639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] avg=65536.0000 max=66192 max_rank=0 min=63534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111518 max_load_reduction_pct=0.627528 moved_to_replicas=151475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] avg=65536.0000 max=66192 max_rank=0 min=63534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111518 max_load_reduction_pct=0.627528 moved_to_replicas=151475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[6423, 58972, 64553, 13943, 30797, 106149, 177710, 65741] avg=65536.0000 max=177710 max_rank=6 min=6423 min_rank=0 imbalance_ratio=2.711639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] avg=65536.0000 max=66192 max_rank=0 min=63534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111518 max_load_reduction_pct=0.627528 moved_to_replicas=151475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63466 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60122 max_load_reduction_pct=0.475973 moved_to_replicas=118609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63466 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60122 max_load_reduction_pct=0.475973 moved_to_replicas=118609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] avg=65536.0000 max=66192 max_rank=0 min=63534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111518 max_load_reduction_pct=0.627528 moved_to_replicas=151475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65116, 494, 124197, 66674, 62984, 21960, 126314, 56549] avg=65536.0000 max=126314 max_rank=6 min=494 min_rank=1 imbalance_ratio=1.927399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[66192, 63534, 64553, 65692, 66192, 66192, 66192, 65741] avg=65536.0000 max=66192 max_rank=0 min=63534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=111518 max_load_reduction_pct=0.627528 moved_to_replicas=151475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65116, 494, 124197, 66674, 62984, 21960, 126314, 56549] avg=65536.0000 max=126314 max_rank=6 min=494 min_rank=1 imbalance_ratio=1.927399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65116, 494, 124197, 66674, 62984, 21960, 126314, 56549] avg=65536.0000 max=126314 max_rank=6 min=494 min_rank=1 imbalance_ratio=1.927399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65116, 494, 124197, 66674, 62984, 21960, 126314, 56549] avg=65536.0000 max=126314 max_rank=6 min=494 min_rank=1 imbalance_ratio=1.927399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63466 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60122 max_load_reduction_pct=0.475973 moved_to_replicas=118609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[6203, 98178, 39366, 110203, 66881, 53787, 13794, 135876] avg=65536.0000 max=135876 max_rank=7 min=6203 min_rank=0 imbalance_ratio=2.073303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[6203, 98178, 39366, 110203, 66881, 53787, 13794, 135876] avg=65536.0000 max=135876 max_rank=7 min=6203 min_rank=0 imbalance_ratio=2.073303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65116, 494, 124197, 66674, 62984, 21960, 126314, 56549] avg=65536.0000 max=126314 max_rank=6 min=494 min_rank=1 imbalance_ratio=1.927399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63466 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60122 max_load_reduction_pct=0.475973 moved_to_replicas=118609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63466 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60122 max_load_reduction_pct=0.475973 moved_to_replicas=118609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65116, 494, 124197, 66674, 62984, 21960, 126314, 56549] avg=65536.0000 max=126314 max_rank=6 min=494 min_rank=1 imbalance_ratio=1.927399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63466 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60122 max_load_reduction_pct=0.475973 moved_to_replicas=118609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69684 max_load_reduction_pct=0.512850 moved_to_replicas=146370 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69684 max_load_reduction_pct=0.512850 moved_to_replicas=146370 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63466 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60122 max_load_reduction_pct=0.475973 moved_to_replicas=118609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[6203, 98178, 39366, 110203, 66881, 53787, 13794, 135876] avg=65536.0000 max=135876 max_rank=7 min=6203 min_rank=0 imbalance_ratio=2.073303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65116, 64746, 66192, 66192, 63466, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63466 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60122 max_load_reduction_pct=0.475973 moved_to_replicas=118609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[6203, 98178, 39366, 110203, 66881, 53787, 13794, 135876] avg=65536.0000 max=135876 max_rank=7 min=6203 min_rank=0 imbalance_ratio=2.073303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[6203, 98178, 39366, 110203, 66881, 53787, 13794, 135876] avg=65536.0000 max=135876 max_rank=7 min=6203 min_rank=0 imbalance_ratio=2.073303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[6203, 98178, 39366, 110203, 66881, 53787, 13794, 135876] avg=65536.0000 max=135876 max_rank=7 min=6203 min_rank=0 imbalance_ratio=2.073303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69684 max_load_reduction_pct=0.512850 moved_to_replicas=146370 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[6203, 98178, 39366, 110203, 66881, 53787, 13794, 135876] avg=65536.0000 max=135876 max_rank=7 min=6203 min_rank=0 imbalance_ratio=2.073303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69684 max_load_reduction_pct=0.512850 moved_to_replicas=146370 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69684 max_load_reduction_pct=0.512850 moved_to_replicas=146370 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[6203, 98178, 39366, 110203, 66881, 53787, 13794, 135876] avg=65536.0000 max=135876 max_rank=7 min=6203 min_rank=0 imbalance_ratio=2.073303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69684 max_load_reduction_pct=0.512850 moved_to_replicas=146370 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69684 max_load_reduction_pct=0.512850 moved_to_replicas=146370 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 60944, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69684 max_load_reduction_pct=0.512850 moved_to_replicas=146370 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=12 total_expected_recv=783610 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=12 total_expected_recv=790215 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=177408 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=12 total_expected_recv=786286 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=181760 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=12 total_expected_recv=793850 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=174336 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=12 total_expected_recv=784041 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=12 total_expected_recv=790151 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=178560 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=12 total_expected_recv=789995 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=179456 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=12 total_expected_recv=773308 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193920 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 15:44:40] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 341.7 | throughput per GPU (TFLOP/s/GPU): 529.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685855E+00 | loss scale: 1.0 | grad norm: 62.781 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=81920 expected_recv=64729 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=81920 expected_recv=62407 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=81920 expected_recv=64729 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=81920 expected_recv=62407 actual_valid_recv=63744 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=81920 expected_recv=62409 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=81920 expected_recv=63632 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65361 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=81920 expected_recv=63632 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65286 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=81920 expected_recv=63623 expected_overflow=0 exact_recv_per_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65241 expected_overflow=0 exact_recv_per_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65361 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65286 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65241 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=81920 expected_recv=63623 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65294 expected_overflow=0 exact_recv_per_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65355 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65248 expected_overflow=0 exact_recv_per_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65294 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65248 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65475 expected_overflow=0 exact_recv_per_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64199 expected_overflow=0 exact_recv_per_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64995 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66097 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65008 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64995 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66097 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64131 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65480 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65008 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64131 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65475 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64199 actual_valid_recv=65536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66107 expected_overflow=0 exact_recv_per_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64965 expected_overflow=0 exact_recv_per_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66107 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64965 expected_overflow=0 exact_recv_per_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64965 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64965 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65948 expected_overflow=0 exact_recv_per_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65948 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65939 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=81920 expected_recv=62533 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65939 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=81920 expected_recv=62511 expected_overflow=0 exact_recv_per_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=81920 expected_recv=62533 actual_valid_recv=63104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=81920 expected_recv=62511 actual_valid_recv=63104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=63104 slack_slots=18816 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64325 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65090 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64325 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65090 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64585 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65077 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65077 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64295 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64585 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65520 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64604 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64295 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=81920 expected_recv=64604 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=81920 expected_recv=61598 expected_overflow=0 exact_recv_per_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=81920 expected_recv=61598 actual_valid_recv=62592 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=81920 expected_recv=61600 expected_overflow=0 exact_recv_per_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=81920 expected_recv=61600 actual_valid_recv=62592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64993 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65836 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64993 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65836 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64343 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64343 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64348 expected_overflow=0 exact_recv_per_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64069 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64348 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64069 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64315 expected_overflow=0 exact_recv_per_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65837 expected_overflow=0 exact_recv_per_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64315 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64971 expected_overflow=0 exact_recv_per_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65837 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64971 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64397 expected_overflow=0 exact_recv_per_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=81920 expected_recv=64397 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=81920 expected_recv=63817 expected_overflow=0 exact_recv_per_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64804 expected_overflow=0 exact_recv_per_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=81920 expected_recv=63817 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64804 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65366 expected_overflow=0 exact_recv_per_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65742 expected_overflow=0 exact_recv_per_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65366 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64595 expected_overflow=0 exact_recv_per_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65114 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64806 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65114 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=81920 expected_recv=64806 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64278 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65573 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64278 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65573 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64595 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65742 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65468 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64621 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64621 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65772 expected_overflow=0 exact_recv_per_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65772 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65583 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64323 expected_overflow=0 exact_recv_per_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=81920 expected_recv=64323 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65469 expected_overflow=0 exact_recv_per_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65469 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65365 expected_overflow=0 exact_recv_per_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65227 expected_overflow=0 exact_recv_per_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65365 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65227 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65385 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=81920 expected_recv=62736 expected_overflow=0 exact_recv_per_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65505 expected_overflow=0 exact_recv_per_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65256 expected_overflow=0 exact_recv_per_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=81920 expected_recv=62736 actual_valid_recv=63360 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=81920 expected_recv=62733 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=81920 expected_recv=62733 actual_valid_recv=63488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65210 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65210 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65360 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65193 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65360 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65193 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65505 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65256 actual_valid_recv=65792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=81920 expected_recv=64993 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63360 slack_slots=18560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65193 expected_overflow=0 exact_recv_per_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65235 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=81920 expected_recv=64993 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65235 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65622 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65611 expected_overflow=0 exact_recv_per_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65193 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65611 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65362 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=81920 expected_recv=64977 expected_overflow=0 exact_recv_per_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66103 expected_overflow=0 exact_recv_per_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65170 expected_overflow=0 exact_recv_per_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=81920 expected_recv=64977 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66103 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65170 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65230 expected_overflow=0 exact_recv_per_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65230 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65101 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66127 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65101 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66127 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=81920 expected_recv=64086 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65106 expected_overflow=0 exact_recv_per_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=81920 expected_recv=64103 expected_overflow=0 exact_recv_per_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65106 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[25960, 193387, 93964, 4768, 53303, 7965, 40230, 104711] avg=65536.0000 max=193387 max_rank=1 min=4768 min_rank=3 imbalance_ratio=2.950851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[25960, 193387, 93964, 4768, 53303, 7965, 40230, 104711] avg=65536.0000 max=193387 max_rank=1 min=4768 min_rank=3 imbalance_ratio=2.950851 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=81920 expected_recv=64103 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[25960, 193387, 93964, 4768, 53303, 7965, 40230, 104711] avg=65536.0000 max=193387 max_rank=1 min=4768 min_rank=3 imbalance_ratio=2.950851 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=81920 expected_recv=64086 actual_valid_recv=64896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65182 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65216 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65182 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65216 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[25901, 193594, 93963, 4748, 53242, 7970, 40089, 104781] avg=65536.0000 max=193594 max_rank=1 min=4748 min_rank=3 imbalance_ratio=2.954010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 64727, 66192, 62409, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=62409 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127401 max_load_reduction_pct=0.658083 moved_to_replicas=193762 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65241, 121288, 236, 124835, 39, 131514, 17503, 63632] avg=65536.0000 max=131514 max_rank=5 min=39 min_rank=4 imbalance_ratio=2.006744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65241, 66193, 65286, 66192, 65361, 66192, 66191, 63632] avg=65536.0000 max=66193 max_rank=1 min=63632 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65321 max_load_reduction_pct=0.496685 moved_to_replicas=179061 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[118343, 65480, 202638, 245, 298, 67994, 3193, 66097] avg=65536.0000 max=202638 max_rank=2 min=245 min_rank=3 imbalance_ratio=3.092010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65480, 66192, 64995, 65008, 66192, 64131, 66097] avg=65536.0000 max=66193 max_rank=0 min=64131 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=136445 max_load_reduction_pct=0.673344 moved_to_replicas=190399 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 186877, 131061, 59225, 64, 134, 15473] avg=65536.0000 max=186877 max_rank=2 min=64 min_rank=5 imbalance_ratio=2.851517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65515, 65939, 66191, 66192, 62533, 65533, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62533 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120683 max_load_reduction_pct=0.645788 moved_to_replicas=185554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[131197, 129459, 383, 17536, 115783, 85, 65520, 64325] avg=65536.0000 max=131197 max_rank=0 min=85 min_rank=5 imbalance_ratio=2.001907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64585, 66194, 66190, 65090, 65520, 64325] avg=65536.0000 max=66194 max_rank=3 min=64325 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65003 max_load_reduction_pct=0.495461 moved_to_replicas=177863 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[78243, 65538, 130551, 69392, 66773, 94359, 12607, 6825] avg=65536.0000 max=130551 max_rank=2 min=6825 min_rank=7 imbalance_ratio=1.992050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[66191, 65538, 66192, 66192, 66192, 66191, 61600, 66192] avg=65536.0000 max=66192 max_rank=2 min=61600 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64359 max_load_reduction_pct=0.492980 moved_to_replicas=108358 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[63951, 108861, 3886, 86221, 129476, 2552, 64348, 64993] avg=65536.0000 max=129476 max_rank=4 min=2552 min_rank=5 imbalance_ratio=1.975647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[64343, 66193, 66191, 66192, 66192, 65836, 64348, 64993] avg=65536.0000 max=66193 max_rank=1 min=64343 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=63283 max_load_reduction_pct=0.488762 moved_to_replicas=125982 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[642, 65531, 44, 54872, 130664, 71973, 130954, 69608] avg=65536.0000 max=130954 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[65114, 65531, 64806, 64069, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64762 max_load_reduction_pct=0.494540 moved_to_replicas=138431 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[131882, 64621, 131058, 63688, 65468, 66782, 674, 115] avg=65536.0000 max=131882 max_rank=0 min=115 min_rank=7 imbalance_ratio=2.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64621, 66192, 64278, 65468, 66192, 65772, 65573] avg=65536.0000 max=66192 max_rank=0 min=64278 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65690 max_load_reduction_pct=0.498097 moved_to_replicas=131146 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[50357, 8104, 451, 129834, 130951, 68622, 70584, 65385] avg=65536.0000 max=130951 max_rank=4 min=451 min_rank=2 imbalance_ratio=1.998154 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[62733, 66192, 65210, 66192, 66192, 66192, 66192, 65385] avg=65536.0000 max=66192 max_rank=1 min=62733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64759 max_load_reduction_pct=0.494528 moved_to_replicas=135223 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[207, 65235, 1153, 65360, 130978, 130661, 65501, 65193] avg=65536.0000 max=130978 max_rank=4 min=207 min_rank=0 imbalance_ratio=1.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[64993, 65235, 65622, 65360, 66192, 66192, 65501, 65193] avg=65536.0000 max=66192 max_rank=4 min=64993 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64786 max_load_reduction_pct=0.494633 moved_to_replicas=129255 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[404, 116, 65216, 198908, 10, 66127, 120764, 72743] avg=65536.0000 max=198908 max_rank=3 min=10 min_rank=4 imbalance_ratio=3.035095 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[64086, 65182, 65216, 66192, 65101, 66127, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64086 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132716 max_load_reduction_pct=0.667223 moved_to_replicas=193839 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=12 total_expected_recv=788338 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182272 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=12 total_expected_recv=786775 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183808 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=12 total_expected_recv=786813 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=12 total_expected_recv=786157 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=12 total_expected_recv=788487 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183296 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=12 total_expected_recv=782986 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188800 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=12 total_expected_recv=784025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186624 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=12 total_expected_recv=787875 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66192 required_capacity=1.010010 - [2026-08-26 14:59:19] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 341.8 | throughput per GPU (TFLOP/s/GPU): 529.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338049E+01 | loss scale: 1.0 | grad norm: 20.321 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[25960, 193387, 93964, 4768, 53303, 7965, 40230, 104711] avg=65536.0000 max=193387 max_rank=1 min=4768 min_rank=3 imbalance_ratio=2.950851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62407 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127194 max_load_reduction_pct=0.657717 moved_to_replicas=193486 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[25960, 193387, 93964, 4768, 53303, 7965, 40230, 104711] avg=65536.0000 max=193387 max_rank=1 min=4768 min_rank=3 imbalance_ratio=2.950851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[25960, 193387, 93964, 4768, 53303, 7965, 40230, 104711] avg=65536.0000 max=193387 max_rank=1 min=4768 min_rank=3 imbalance_ratio=2.950851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62407 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127194 max_load_reduction_pct=0.657717 moved_to_replicas=193486 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62407 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127194 max_load_reduction_pct=0.657717 moved_to_replicas=193486 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[25960, 193387, 93964, 4768, 53303, 7965, 40230, 104711] avg=65536.0000 max=193387 max_rank=1 min=4768 min_rank=3 imbalance_ratio=2.950851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62407 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127194 max_load_reduction_pct=0.657717 moved_to_replicas=193486 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62407 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127194 max_load_reduction_pct=0.657717 moved_to_replicas=193486 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[25960, 193387, 93964, 4768, 53303, 7965, 40230, 104711] avg=65536.0000 max=193387 max_rank=1 min=4768 min_rank=3 imbalance_ratio=2.950851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62407 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127194 max_load_reduction_pct=0.657717 moved_to_replicas=193486 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65248, 120767, 233, 124655, 42, 131505, 18215, 63623] avg=65536.0000 max=131505 max_rank=5 min=42 min_rank=4 imbalance_ratio=2.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65248, 120767, 233, 124655, 42, 131505, 18215, 63623] avg=65536.0000 max=131505 max_rank=5 min=42 min_rank=4 imbalance_ratio=2.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65248, 120767, 233, 124655, 42, 131505, 18215, 63623] avg=65536.0000 max=131505 max_rank=5 min=42 min_rank=4 imbalance_ratio=2.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62407 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127194 max_load_reduction_pct=0.657717 moved_to_replicas=193486 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[66192, 66192, 66193, 64729, 66192, 62407, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62407 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=127194 max_load_reduction_pct=0.657717 moved_to_replicas=193486 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] avg=65536.0000 max=66193 max_rank=1 min=63623 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65312 max_load_reduction_pct=0.496650 moved_to_replicas=178351 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65248, 120767, 233, 124655, 42, 131505, 18215, 63623] avg=65536.0000 max=131505 max_rank=5 min=42 min_rank=4 imbalance_ratio=2.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65248, 120767, 233, 124655, 42, 131505, 18215, 63623] avg=65536.0000 max=131505 max_rank=5 min=42 min_rank=4 imbalance_ratio=2.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] avg=65536.0000 max=66193 max_rank=1 min=63623 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65312 max_load_reduction_pct=0.496650 moved_to_replicas=178351 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65248, 120767, 233, 124655, 42, 131505, 18215, 63623] avg=65536.0000 max=131505 max_rank=5 min=42 min_rank=4 imbalance_ratio=2.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] avg=65536.0000 max=66193 max_rank=1 min=63623 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65312 max_load_reduction_pct=0.496650 moved_to_replicas=178351 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] avg=65536.0000 max=66193 max_rank=1 min=63623 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65312 max_load_reduction_pct=0.496650 moved_to_replicas=178351 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65248, 120767, 233, 124655, 42, 131505, 18215, 63623] avg=65536.0000 max=131505 max_rank=5 min=42 min_rank=4 imbalance_ratio=2.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] avg=65536.0000 max=66193 max_rank=1 min=63623 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65312 max_load_reduction_pct=0.496650 moved_to_replicas=178351 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65248, 120767, 233, 124655, 42, 131505, 18215, 63623] avg=65536.0000 max=131505 max_rank=5 min=42 min_rank=4 imbalance_ratio=2.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] avg=65536.0000 max=66193 max_rank=1 min=63623 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65312 max_load_reduction_pct=0.496650 moved_to_replicas=178351 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[117030, 65475, 203707, 231, 296, 68191, 3251, 66107] avg=65536.0000 max=203707 max_rank=2 min=231 min_rank=3 imbalance_ratio=3.108322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[117030, 65475, 203707, 231, 296, 68191, 3251, 66107] avg=65536.0000 max=203707 max_rank=2 min=231 min_rank=3 imbalance_ratio=3.108322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[117030, 65475, 203707, 231, 296, 68191, 3251, 66107] avg=65536.0000 max=203707 max_rank=2 min=231 min_rank=3 imbalance_ratio=3.108322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] avg=65536.0000 max=66193 max_rank=1 min=63623 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65312 max_load_reduction_pct=0.496650 moved_to_replicas=178351 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65248, 66193, 65294, 66192, 65355, 66192, 66191, 63623] avg=65536.0000 max=66193 max_rank=1 min=63623 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=65312 max_load_reduction_pct=0.496650 moved_to_replicas=178351 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] avg=65536.0000 max=66193 max_rank=0 min=64199 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=137514 max_load_reduction_pct=0.675058 moved_to_replicas=190352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] avg=65536.0000 max=66193 max_rank=0 min=64199 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=137514 max_load_reduction_pct=0.675058 moved_to_replicas=190352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[117030, 65475, 203707, 231, 296, 68191, 3251, 66107] avg=65536.0000 max=203707 max_rank=2 min=231 min_rank=3 imbalance_ratio=3.108322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[117030, 65475, 203707, 231, 296, 68191, 3251, 66107] avg=65536.0000 max=203707 max_rank=2 min=231 min_rank=3 imbalance_ratio=3.108322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] avg=65536.0000 max=66193 max_rank=0 min=64199 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=137514 max_load_reduction_pct=0.675058 moved_to_replicas=190352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[117030, 65475, 203707, 231, 296, 68191, 3251, 66107] avg=65536.0000 max=203707 max_rank=2 min=231 min_rank=3 imbalance_ratio=3.108322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] avg=65536.0000 max=66193 max_rank=0 min=64199 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=137514 max_load_reduction_pct=0.675058 moved_to_replicas=190352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[117030, 65475, 203707, 231, 296, 68191, 3251, 66107] avg=65536.0000 max=203707 max_rank=2 min=231 min_rank=3 imbalance_ratio=3.108322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 186556, 131060, 59544, 65, 136, 15453] avg=65536.0000 max=186556 max_rank=2 min=65 min_rank=5 imbalance_ratio=2.846619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] avg=65536.0000 max=66193 max_rank=0 min=64199 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=137514 max_load_reduction_pct=0.675058 moved_to_replicas=190352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[117030, 65475, 203707, 231, 296, 68191, 3251, 66107] avg=65536.0000 max=203707 max_rank=2 min=231 min_rank=3 imbalance_ratio=3.108322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] avg=65536.0000 max=66193 max_rank=0 min=64199 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=137514 max_load_reduction_pct=0.675058 moved_to_replicas=190352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 186556, 131060, 59544, 65, 136, 15453] avg=65536.0000 max=186556 max_rank=2 min=65 min_rank=5 imbalance_ratio=2.846619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 186556, 131060, 59544, 65, 136, 15453] avg=65536.0000 max=186556 max_rank=2 min=65 min_rank=5 imbalance_ratio=2.846619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62511 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120362 max_load_reduction_pct=0.645179 moved_to_replicas=185232 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] avg=65536.0000 max=66193 max_rank=0 min=64199 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=137514 max_load_reduction_pct=0.675058 moved_to_replicas=190352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[66193, 65475, 66192, 64965, 64965, 66192, 64199, 66107] avg=65536.0000 max=66193 max_rank=0 min=64199 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=137514 max_load_reduction_pct=0.675058 moved_to_replicas=190352 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62511 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120362 max_load_reduction_pct=0.645179 moved_to_replicas=185232 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 186556, 131060, 59544, 65, 136, 15453] avg=65536.0000 max=186556 max_rank=2 min=65 min_rank=5 imbalance_ratio=2.846619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 186556, 131060, 59544, 65, 136, 15453] avg=65536.0000 max=186556 max_rank=2 min=65 min_rank=5 imbalance_ratio=2.846619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62511 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120362 max_load_reduction_pct=0.645179 moved_to_replicas=185232 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 186556, 131060, 59544, 65, 136, 15453] avg=65536.0000 max=186556 max_rank=2 min=65 min_rank=5 imbalance_ratio=2.846619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62511 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120362 max_load_reduction_pct=0.645179 moved_to_replicas=185232 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[131187, 129446, 392, 17306, 116036, 82, 65544, 64295] avg=65536.0000 max=131187 max_rank=0 min=82 min_rank=5 imbalance_ratio=2.001755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62511 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120362 max_load_reduction_pct=0.645179 moved_to_replicas=185232 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 186556, 131060, 59544, 65, 136, 15453] avg=65536.0000 max=186556 max_rank=2 min=65 min_rank=5 imbalance_ratio=2.846619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62511 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120362 max_load_reduction_pct=0.645179 moved_to_replicas=185232 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[131187, 129446, 392, 17306, 116036, 82, 65544, 64295] avg=65536.0000 max=131187 max_rank=0 min=82 min_rank=5 imbalance_ratio=2.001755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 186556, 131060, 59544, 65, 136, 15453] avg=65536.0000 max=186556 max_rank=2 min=65 min_rank=5 imbalance_ratio=2.846619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[131187, 129446, 392, 17306, 116036, 82, 65544, 64295] avg=65536.0000 max=131187 max_rank=0 min=82 min_rank=5 imbalance_ratio=2.001755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] avg=65536.0000 max=66192 max_rank=0 min=64295 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64995 max_load_reduction_pct=0.495438 moved_to_replicas=178093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62511 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120362 max_load_reduction_pct=0.645179 moved_to_replicas=185232 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[131187, 129446, 392, 17306, 116036, 82, 65544, 64295] avg=65536.0000 max=131187 max_rank=0 min=82 min_rank=5 imbalance_ratio=2.001755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[131187, 129446, 392, 17306, 116036, 82, 65544, 64295] avg=65536.0000 max=131187 max_rank=0 min=82 min_rank=5 imbalance_ratio=2.001755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] avg=65536.0000 max=66192 max_rank=0 min=64295 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64995 max_load_reduction_pct=0.495438 moved_to_replicas=178093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65526, 65948, 66191, 66192, 62511, 65535, 66194, 66191] avg=65536.0000 max=66194 max_rank=6 min=62511 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=120362 max_load_reduction_pct=0.645179 moved_to_replicas=185232 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] avg=65536.0000 max=66192 max_rank=0 min=64295 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64995 max_load_reduction_pct=0.495438 moved_to_replicas=178093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[131187, 129446, 392, 17306, 116036, 82, 65544, 64295] avg=65536.0000 max=131187 max_rank=0 min=82 min_rank=5 imbalance_ratio=2.001755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[78274, 65540, 130559, 69390, 66821, 94446, 12693, 6565] avg=65536.0000 max=130559 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.992172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] avg=65536.0000 max=66192 max_rank=0 min=64295 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64995 max_load_reduction_pct=0.495438 moved_to_replicas=178093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] avg=65536.0000 max=66192 max_rank=0 min=64295 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64995 max_load_reduction_pct=0.495438 moved_to_replicas=178093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[131187, 129446, 392, 17306, 116036, 82, 65544, 64295] avg=65536.0000 max=131187 max_rank=0 min=82 min_rank=5 imbalance_ratio=2.001755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] avg=65536.0000 max=66192 max_rank=0 min=64295 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64995 max_load_reduction_pct=0.495438 moved_to_replicas=178093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[78274, 65540, 130559, 69390, 66821, 94446, 12693, 6565] avg=65536.0000 max=130559 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.992172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[131187, 129446, 392, 17306, 116036, 82, 65544, 64295] avg=65536.0000 max=131187 max_rank=0 min=82 min_rank=5 imbalance_ratio=2.001755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[78274, 65540, 130559, 69390, 66821, 94446, 12693, 6565] avg=65536.0000 max=130559 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.992172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] avg=65536.0000 max=66192 max_rank=2 min=61598 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64367 max_load_reduction_pct=0.493011 moved_to_replicas=108530 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] avg=65536.0000 max=66192 max_rank=0 min=64295 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64995 max_load_reduction_pct=0.495438 moved_to_replicas=178093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[78274, 65540, 130559, 69390, 66821, 94446, 12693, 6565] avg=65536.0000 max=130559 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.992172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] avg=65536.0000 max=66192 max_rank=2 min=61598 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64367 max_load_reduction_pct=0.493011 moved_to_replicas=108530 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[78274, 65540, 130559, 69390, 66821, 94446, 12693, 6565] avg=65536.0000 max=130559 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.992172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[66192, 66192, 64604, 66192, 66192, 65077, 65544, 64295] avg=65536.0000 max=66192 max_rank=0 min=64295 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64995 max_load_reduction_pct=0.495438 moved_to_replicas=178093 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] avg=65536.0000 max=66192 max_rank=2 min=61598 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64367 max_load_reduction_pct=0.493011 moved_to_replicas=108530 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[78274, 65540, 130559, 69390, 66821, 94446, 12693, 6565] avg=65536.0000 max=130559 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.992172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[64011, 107981, 3936, 87045, 129505, 2524, 64315, 64971] avg=65536.0000 max=129505 max_rank=4 min=2524 min_rank=5 imbalance_ratio=1.976089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] avg=65536.0000 max=66192 max_rank=2 min=61598 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64367 max_load_reduction_pct=0.493011 moved_to_replicas=108530 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[78274, 65540, 130559, 69390, 66821, 94446, 12693, 6565] avg=65536.0000 max=130559 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.992172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] avg=65536.0000 max=66192 max_rank=2 min=61598 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64367 max_load_reduction_pct=0.493011 moved_to_replicas=108530 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] avg=65536.0000 max=66192 max_rank=2 min=61598 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64367 max_load_reduction_pct=0.493011 moved_to_replicas=108530 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] avg=65536.0000 max=66193 max_rank=2 min=64315 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=63312 max_load_reduction_pct=0.488877 moved_to_replicas=125955 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[64011, 107981, 3936, 87045, 129505, 2524, 64315, 64971] avg=65536.0000 max=129505 max_rank=4 min=2524 min_rank=5 imbalance_ratio=1.976089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[78274, 65540, 130559, 69390, 66821, 94446, 12693, 6565] avg=65536.0000 max=130559 max_rank=2 min=6565 min_rank=7 imbalance_ratio=1.992172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[64011, 107981, 3936, 87045, 129505, 2524, 64315, 64971] avg=65536.0000 max=129505 max_rank=4 min=2524 min_rank=5 imbalance_ratio=1.976089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] avg=65536.0000 max=66192 max_rank=2 min=61598 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64367 max_load_reduction_pct=0.493011 moved_to_replicas=108530 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[64011, 107981, 3936, 87045, 129505, 2524, 64315, 64971] avg=65536.0000 max=129505 max_rank=4 min=2524 min_rank=5 imbalance_ratio=1.976089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] avg=65536.0000 max=66193 max_rank=2 min=64315 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=63312 max_load_reduction_pct=0.488877 moved_to_replicas=125955 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[64011, 107981, 3936, 87045, 129505, 2524, 64315, 64971] avg=65536.0000 max=129505 max_rank=4 min=2524 min_rank=5 imbalance_ratio=1.976089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[66190, 65540, 66192, 66192, 66192, 66192, 61598, 66192] avg=65536.0000 max=66192 max_rank=2 min=61598 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64367 max_load_reduction_pct=0.493011 moved_to_replicas=108530 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] avg=65536.0000 max=66193 max_rank=2 min=64315 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=63312 max_load_reduction_pct=0.488877 moved_to_replicas=125955 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[64011, 107981, 3936, 87045, 129505, 2524, 64315, 64971] avg=65536.0000 max=129505 max_rank=4 min=2524 min_rank=5 imbalance_ratio=1.976089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[653, 65533, 44, 55047, 130905, 71936, 130952, 69218] avg=65536.0000 max=130952 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] avg=65536.0000 max=66193 max_rank=2 min=64315 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=63312 max_load_reduction_pct=0.488877 moved_to_replicas=125955 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[64011, 107981, 3936, 87045, 129505, 2524, 64315, 64971] avg=65536.0000 max=129505 max_rank=4 min=2524 min_rank=5 imbalance_ratio=1.976089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] avg=65536.0000 max=66193 max_rank=2 min=64315 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=63312 max_load_reduction_pct=0.488877 moved_to_replicas=125955 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] avg=65536.0000 max=66193 max_rank=2 min=64315 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=63312 max_load_reduction_pct=0.488877 moved_to_replicas=125955 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63817 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64760 max_load_reduction_pct=0.494532 moved_to_replicas=138243 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[653, 65533, 44, 55047, 130905, 71936, 130952, 69218] avg=65536.0000 max=130952 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[64011, 107981, 3936, 87045, 129505, 2524, 64315, 64971] avg=65536.0000 max=129505 max_rank=4 min=2524 min_rank=5 imbalance_ratio=1.976089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] avg=65536.0000 max=66193 max_rank=2 min=64315 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=63312 max_load_reduction_pct=0.488877 moved_to_replicas=125955 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[653, 65533, 44, 55047, 130905, 71936, 130952, 69218] avg=65536.0000 max=130952 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[653, 65533, 44, 55047, 130905, 71936, 130952, 69218] avg=65536.0000 max=130952 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[653, 65533, 44, 55047, 130905, 71936, 130952, 69218] avg=65536.0000 max=130952 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63817 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64760 max_load_reduction_pct=0.494532 moved_to_replicas=138243 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[64397, 66191, 66193, 66192, 66192, 65837, 64315, 64971] avg=65536.0000 max=66193 max_rank=2 min=64315 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=63312 max_load_reduction_pct=0.488877 moved_to_replicas=125955 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[131847, 64595, 131055, 63729, 65469, 66786, 684, 123] avg=65536.0000 max=131847 max_rank=0 min=123 min_rank=7 imbalance_ratio=2.011826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63817 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64760 max_load_reduction_pct=0.494532 moved_to_replicas=138243 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[653, 65533, 44, 55047, 130905, 71936, 130952, 69218] avg=65536.0000 max=130952 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63817 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64760 max_load_reduction_pct=0.494532 moved_to_replicas=138243 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[653, 65533, 44, 55047, 130905, 71936, 130952, 69218] avg=65536.0000 max=130952 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63817 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64760 max_load_reduction_pct=0.494532 moved_to_replicas=138243 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] avg=65536.0000 max=66192 max_rank=0 min=64323 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65655 max_load_reduction_pct=0.497964 moved_to_replicas=131112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63817 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64760 max_load_reduction_pct=0.494532 moved_to_replicas=138243 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[131847, 64595, 131055, 63729, 65469, 66786, 684, 123] avg=65536.0000 max=131847 max_rank=0 min=123 min_rank=7 imbalance_ratio=2.011826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[653, 65533, 44, 55047, 130905, 71936, 130952, 69218] avg=65536.0000 max=130952 max_rank=6 min=44 min_rank=2 imbalance_ratio=1.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[131847, 64595, 131055, 63729, 65469, 66786, 684, 123] avg=65536.0000 max=131847 max_rank=0 min=123 min_rank=7 imbalance_ratio=2.011826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63817 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64760 max_load_reduction_pct=0.494532 moved_to_replicas=138243 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[131847, 64595, 131055, 63729, 65469, 66786, 684, 123] avg=65536.0000 max=131847 max_rank=0 min=123 min_rank=7 imbalance_ratio=2.011826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] avg=65536.0000 max=66192 max_rank=0 min=64323 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65655 max_load_reduction_pct=0.497964 moved_to_replicas=131112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[65366, 65533, 64804, 63817, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63817 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64760 max_load_reduction_pct=0.494532 moved_to_replicas=138243 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[131847, 64595, 131055, 63729, 65469, 66786, 684, 123] avg=65536.0000 max=131847 max_rank=0 min=123 min_rank=7 imbalance_ratio=2.011826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[49926, 7922, 462, 129902, 130957, 68499, 71255, 65365] avg=65536.0000 max=130957 max_rank=4 min=462 min_rank=2 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] avg=65536.0000 max=66192 max_rank=0 min=64323 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65655 max_load_reduction_pct=0.497964 moved_to_replicas=131112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[131847, 64595, 131055, 63729, 65469, 66786, 684, 123] avg=65536.0000 max=131847 max_rank=0 min=123 min_rank=7 imbalance_ratio=2.011826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] avg=65536.0000 max=66192 max_rank=0 min=64323 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65655 max_load_reduction_pct=0.497964 moved_to_replicas=131112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[131847, 64595, 131055, 63729, 65469, 66786, 684, 123] avg=65536.0000 max=131847 max_rank=0 min=123 min_rank=7 imbalance_ratio=2.011826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] avg=65536.0000 max=66192 max_rank=1 min=62736 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=135845 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] avg=65536.0000 max=66192 max_rank=0 min=64323 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65655 max_load_reduction_pct=0.497964 moved_to_replicas=131112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] avg=65536.0000 max=66192 max_rank=0 min=64323 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65655 max_load_reduction_pct=0.497964 moved_to_replicas=131112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[49926, 7922, 462, 129902, 130957, 68499, 71255, 65365] avg=65536.0000 max=130957 max_rank=4 min=462 min_rank=2 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[131847, 64595, 131055, 63729, 65469, 66786, 684, 123] avg=65536.0000 max=131847 max_rank=0 min=123 min_rank=7 imbalance_ratio=2.011826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] avg=65536.0000 max=66192 max_rank=0 min=64323 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65655 max_load_reduction_pct=0.497964 moved_to_replicas=131112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[49926, 7922, 462, 129902, 130957, 68499, 71255, 65365] avg=65536.0000 max=130957 max_rank=4 min=462 min_rank=2 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[49926, 7922, 462, 129902, 130957, 68499, 71255, 65365] avg=65536.0000 max=130957 max_rank=4 min=462 min_rank=2 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] avg=65536.0000 max=66192 max_rank=1 min=62736 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=135845 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[66192, 64595, 66192, 64323, 65469, 66192, 65742, 65583] avg=65536.0000 max=66192 max_rank=0 min=64323 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65655 max_load_reduction_pct=0.497964 moved_to_replicas=131112 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[212, 65256, 1147, 65362, 130957, 130656, 65505, 65193] avg=65536.0000 max=130957 max_rank=4 min=212 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[49926, 7922, 462, 129902, 130957, 68499, 71255, 65365] avg=65536.0000 max=130957 max_rank=4 min=462 min_rank=2 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] avg=65536.0000 max=66192 max_rank=1 min=62736 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=135845 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] avg=65536.0000 max=66192 max_rank=1 min=62736 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=135845 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[49926, 7922, 462, 129902, 130957, 68499, 71255, 65365] avg=65536.0000 max=130957 max_rank=4 min=462 min_rank=2 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[49926, 7922, 462, 129902, 130957, 68499, 71255, 65365] avg=65536.0000 max=130957 max_rank=4 min=462 min_rank=2 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] avg=65536.0000 max=66192 max_rank=4 min=64977 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=129229 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] avg=65536.0000 max=66192 max_rank=1 min=62736 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=135845 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] avg=65536.0000 max=66192 max_rank=1 min=62736 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=135845 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[212, 65256, 1147, 65362, 130957, 130656, 65505, 65193] avg=65536.0000 max=130957 max_rank=4 min=212 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] avg=65536.0000 max=66192 max_rank=1 min=62736 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=135845 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[49926, 7922, 462, 129902, 130957, 68499, 71255, 65365] avg=65536.0000 max=130957 max_rank=4 min=462 min_rank=2 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[212, 65256, 1147, 65362, 130957, 130656, 65505, 65193] avg=65536.0000 max=130957 max_rank=4 min=212 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[212, 65256, 1147, 65362, 130957, 130656, 65505, 65193] avg=65536.0000 max=130957 max_rank=4 min=212 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] avg=65536.0000 max=66192 max_rank=4 min=64977 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=129229 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[401, 109, 65230, 198810, 12, 66103, 120337, 73286] avg=65536.0000 max=198810 max_rank=3 min=12 min_rank=4 imbalance_ratio=3.033600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[62736, 66192, 65227, 66192, 66192, 66192, 66192, 65365] avg=65536.0000 max=66192 max_rank=1 min=62736 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=135845 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] avg=65536.0000 max=66192 max_rank=4 min=64977 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=129229 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[212, 65256, 1147, 65362, 130957, 130656, 65505, 65193] avg=65536.0000 max=130957 max_rank=4 min=212 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[212, 65256, 1147, 65362, 130957, 130656, 65505, 65193] avg=65536.0000 max=130957 max_rank=4 min=212 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] avg=65536.0000 max=66192 max_rank=4 min=64977 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=129229 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[212, 65256, 1147, 65362, 130957, 130656, 65505, 65193] avg=65536.0000 max=130957 max_rank=4 min=212 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64103 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132618 max_load_reduction_pct=0.667059 moved_to_replicas=193857 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] avg=65536.0000 max=66192 max_rank=4 min=64977 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=129229 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] avg=65536.0000 max=66192 max_rank=4 min=64977 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=129229 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[401, 109, 65230, 198810, 12, 66103, 120337, 73286] avg=65536.0000 max=198810 max_rank=3 min=12 min_rank=4 imbalance_ratio=3.033600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] avg=65536.0000 max=66192 max_rank=4 min=64977 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=129229 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[212, 65256, 1147, 65362, 130957, 130656, 65505, 65193] avg=65536.0000 max=130957 max_rank=4 min=212 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[401, 109, 65230, 198810, 12, 66103, 120337, 73286] avg=65536.0000 max=198810 max_rank=3 min=12 min_rank=4 imbalance_ratio=3.033600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[401, 109, 65230, 198810, 12, 66103, 120337, 73286] avg=65536.0000 max=198810 max_rank=3 min=12 min_rank=4 imbalance_ratio=3.033600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64103 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132618 max_load_reduction_pct=0.667059 moved_to_replicas=193857 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[64977, 65256, 65611, 65362, 66192, 66192, 65505, 65193] avg=65536.0000 max=66192 max_rank=4 min=64977 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=129229 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64103 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132618 max_load_reduction_pct=0.667059 moved_to_replicas=193857 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[401, 109, 65230, 198810, 12, 66103, 120337, 73286] avg=65536.0000 max=198810 max_rank=3 min=12 min_rank=4 imbalance_ratio=3.033600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[401, 109, 65230, 198810, 12, 66103, 120337, 73286] avg=65536.0000 max=198810 max_rank=3 min=12 min_rank=4 imbalance_ratio=3.033600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[401, 109, 65230, 198810, 12, 66103, 120337, 73286] avg=65536.0000 max=198810 max_rank=3 min=12 min_rank=4 imbalance_ratio=3.033600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64103 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132618 max_load_reduction_pct=0.667059 moved_to_replicas=193857 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64103 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132618 max_load_reduction_pct=0.667059 moved_to_replicas=193857 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64103 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132618 max_load_reduction_pct=0.667059 moved_to_replicas=193857 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[401, 109, 65230, 198810, 12, 66103, 120337, 73286] avg=65536.0000 max=198810 max_rank=3 min=12 min_rank=4 imbalance_ratio=3.033600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64103 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132618 max_load_reduction_pct=0.667059 moved_to_replicas=193857 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[64103, 65170, 65230, 66192, 65106, 66103, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64103 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=132618 max_load_reduction_pct=0.667059 moved_to_replicas=193857 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=12 total_expected_recv=784055 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186112 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=12 total_expected_recv=787923 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=12 total_expected_recv=788303 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=181888 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=12 total_expected_recv=788477 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183424 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=12 total_expected_recv=786540 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=12 total_expected_recv=786750 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=12 total_expected_recv=786096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185600 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=12 total_expected_recv=783312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188288 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 15:44:40] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 342.6 | throughput per GPU (TFLOP/s/GPU): 527.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338075E+01 | loss scale: 1.0 | grad norm: 20.308 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=81920 expected_recv=62587 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=81920 expected_recv=62572 expected_overflow=0 exact_recv_per_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=81920 expected_recv=62587 actual_valid_recv=63744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=81920 expected_recv=62572 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65232 expected_overflow=0 exact_recv_per_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67968 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65218 expected_overflow=0 exact_recv_per_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65232 actual_valid_recv=66432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65218 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65574 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65506 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=81920 expected_recv=64627 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65239 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65239 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65613 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=81920 expected_recv=64627 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65231 expected_overflow=0 exact_recv_per_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65345 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65231 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=81920 expected_recv=64641 expected_overflow=0 exact_recv_per_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65610 expected_overflow=0 exact_recv_per_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65344 expected_overflow=0 exact_recv_per_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=81920 expected_recv=64641 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65503 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65610 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65575 actual_valid_recv=66944 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65345 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65344 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=81920 expected_recv=63637 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65500 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=81920 expected_recv=64190 expected_overflow=0 exact_recv_per_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=81920 expected_recv=63614 expected_overflow=0 exact_recv_per_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=81920 expected_recv=63637 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=81920 expected_recv=63614 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=81920 expected_recv=64214 expected_overflow=0 exact_recv_per_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=81920 expected_recv=64214 actual_valid_recv=65152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=81920 expected_recv=64190 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65458 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65458 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=81920 expected_recv=62994 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=81920 expected_recv=62994 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64814 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=81920 expected_recv=62993 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65460 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=81920 expected_recv=62993 actual_valid_recv=64000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64539 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64539 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64814 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64636 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64811 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64639 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64529 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64636 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64811 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65645 expected_overflow=0 exact_recv_per_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64639 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64529 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65566 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65645 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=81920 expected_recv=63590 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=81920 expected_recv=63590 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65644 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65386 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65644 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=81920 expected_recv=63844 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65386 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=81920 expected_recv=63844 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65373 expected_overflow=0 exact_recv_per_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65373 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=81920 expected_recv=63596 expected_overflow=0 exact_recv_per_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65816 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65816 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=81920 expected_recv=64324 expected_overflow=0 exact_recv_per_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=81920 expected_recv=63596 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65568 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=81920 expected_recv=64324 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65585 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64334 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64334 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65497 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65497 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65481 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64815 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65481 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64815 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65785 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65331 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65785 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=81920 expected_recv=64283 expected_overflow=0 exact_recv_per_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=81920 expected_recv=64283 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65821 expected_overflow=0 exact_recv_per_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=81920 expected_recv=63881 expected_overflow=0 exact_recv_per_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65821 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64682 expected_overflow=0 exact_recv_per_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=81920 expected_recv=63881 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64682 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65481 expected_overflow=0 exact_recv_per_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65481 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64377 expected_overflow=0 exact_recv_per_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65697 expected_overflow=0 exact_recv_per_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64377 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65476 expected_overflow=0 exact_recv_per_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65697 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65476 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65331 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65380 expected_overflow=0 exact_recv_per_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65344 expected_overflow=0 exact_recv_per_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65380 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65344 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65772 expected_overflow=0 exact_recv_per_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=81920 expected_recv=63677 expected_overflow=0 exact_recv_per_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65772 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65253 expected_overflow=0 exact_recv_per_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=81920 expected_recv=64312 expected_overflow=0 exact_recv_per_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=81920 expected_recv=63677 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65253 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65162 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=81920 expected_recv=63893 expected_overflow=0 exact_recv_per_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=81920 expected_recv=64312 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65879 expected_overflow=0 exact_recv_per_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65162 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=81920 expected_recv=63893 actual_valid_recv=64512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=81920 expected_recv=64318 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65857 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=81920 expected_recv=64318 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65857 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65258 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65378 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65258 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64545 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65378 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65580 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64545 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64649 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65582 expected_overflow=0 exact_recv_per_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65879 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65582 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64649 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65365 expected_overflow=0 exact_recv_per_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65681 expected_overflow=0 exact_recv_per_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65365 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65681 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65345 expected_overflow=0 exact_recv_per_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65677 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65345 actual_valid_recv=65920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66139 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65677 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65360 expected_overflow=0 exact_recv_per_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66139 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65360 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[30391, 185404, 81235, 2327, 42380, 6657, 58236, 117658] avg=65536.0000 max=185404 max_rank=1 min=2327 min_rank=3 imbalance_ratio=2.829041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119211 max_load_reduction_pct=0.642980 moved_to_replicas=185721 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65218, 131278, 120, 68862, 15, 138815, 43008, 76972] avg=65536.0000 max=138815 max_rank=5 min=15 min_rank=4 imbalance_ratio=2.118149 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65218, 66192, 66192, 66192, 65523, 66192, 62587, 66192] avg=65536.0000 max=66192 max_rank=1 min=62587 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=72623 max_load_reduction_pct=0.523164 moved_to_replicas=151159 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65345, 131409, 65506, 65574, 22, 129950, 869, 65613] avg=65536.0000 max=131409 max_rank=1 min=22 min_rank=4 imbalance_ratio=2.005142 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65345, 66192, 65506, 65574, 65239, 66192, 64627, 65613] avg=65536.0000 max=66192 max_rank=1 min=64627 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65217 max_load_reduction_pct=0.496290 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[64190, 195729, 59801, 6284, 4853, 41, 62081, 131309] avg=65536.0000 max=195729 max_rank=1 min=41 min_rank=5 imbalance_ratio=2.986588 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[64190, 66192, 66192, 66192, 66192, 65501, 63637, 66192] avg=65536.0000 max=66192 max_rank=1 min=63637 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129537 max_load_reduction_pct=0.661818 moved_to_replicas=194654 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[140262, 130953, 133, 65538, 81799, 21, 40053, 65529] avg=65536.0000 max=140262 max_rank=0 min=21 min_rank=5 imbalance_ratio=2.140228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65538, 66192, 65460, 62993, 65529] avg=65536.0000 max=66194 max_rank=0 min=62993 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=74068 max_load_reduction_pct=0.528069 moved_to_replicas=154438 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[130634, 130775, 129812, 49014, 65534, 369, 18097, 53] avg=65536.0000 max=130775 max_rank=1 min=53 min_rank=7 imbalance_ratio=1.995468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64539, 65534, 64811, 66192, 64636] avg=65536.0000 max=66192 max_rank=0 min=64539 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64583 max_load_reduction_pct=0.493848 moved_to_replicas=192645 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63402, 126428, 70, 137718, 65644, 65386, 114] avg=65536.0000 max=137718 max_rank=4 min=70 min_rank=3 imbalance_ratio=2.101410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65526, 63590, 66192, 65566, 66192, 65644, 65386, 66192] avg=65536.0000 max=66192 max_rank=2 min=63590 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=71526 max_load_reduction_pct=0.519366 moved_to_replicas=131762 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[2327, 65816, 57367, 102, 28, 195442, 62298, 140908] avg=65536.0000 max=195442 max_rank=5 min=28 min_rank=4 imbalance_ratio=2.982208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65816, 66191, 63844, 65536, 66192, 64324, 66193] avg=65536.0000 max=66193 max_rank=7 min=63844 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=129249 max_load_reduction_pct=0.661316 moved_to_replicas=203966 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[130081, 64, 65497, 27548, 104229, 64815, 66573, 65481] avg=65536.0000 max=130081 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.984879 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64334, 65497, 65585, 66192, 64815, 66192, 65481] avg=65536.0000 max=66192 max_rank=0 min=64334 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63889 max_load_reduction_pct=0.491148 moved_to_replicas=102307 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[63321, 65785, 65331, 65541, 130389, 67751, 63893, 2277] avg=65536.0000 max=130389 max_rank=4 min=2277 min_rank=7 imbalance_ratio=1.989578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65162, 65785, 65331, 65541, 66192, 66192, 63893, 66192] avg=65536.0000 max=66192 max_rank=4 min=63893 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64197 max_load_reduction_pct=0.492350 moved_to_replicas=65756 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65378, 82225, 49824, 195941, 65, 65258, 65580, 17] avg=65536.0000 max=195941 max_rank=3 min=17 min_rank=7 imbalance_ratio=2.989822 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65378, 66192, 65857, 66192, 64318, 65258, 65580, 65513] avg=65536.0000 max=66192 max_rank=1 min=64318 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129749 max_load_reduction_pct=0.662184 moved_to_replicas=145782 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[66139, 414, 130118, 619, 130427, 65677, 65360, 65534] avg=65536.0000 max=130427 max_rank=4 min=414 min_rank=1 imbalance_ratio=1.990158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[66139, 64649, 66192, 64545, 66192, 65677, 65360, 65534] avg=65536.0000 max=66192 max_rank=2 min=64545 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64235 max_load_reduction_pct=0.492498 moved_to_replicas=128161 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=12 total_expected_recv=787517 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=12 total_expected_recv=788126 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183680 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=12 total_expected_recv=789494 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183424 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=12 total_expected_recv=785500 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=12 total_expected_recv=789459 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182912 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=12 total_expected_recv=791724 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=179968 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=12 total_expected_recv=771715 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=201728 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=12 total_expected_recv=787921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66194 required_capacity=1.010040 - [2026-08-26 14:59:19] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 340.1 | throughput per GPU (TFLOP/s/GPU): 531.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.052464E+01 | loss scale: 1.0 | grad norm: 95.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=81920 expected_recv=62846 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=81920 expected_recv=64290 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64610 expected_overflow=0 exact_recv_per_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66176 expected_overflow=0 exact_recv_per_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64610 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64556 expected_overflow=0 exact_recv_per_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=81920 expected_recv=66176 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=81920 expected_recv=64556 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[30884, 185367, 81087, 2399, 42281, 6603, 58158, 117509] avg=65536.0000 max=185367 max_rank=1 min=2399 min_rank=3 imbalance_ratio=2.828476 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[30884, 185367, 81087, 2399, 42281, 6603, 58158, 117509] avg=65536.0000 max=185367 max_rank=1 min=2399 min_rank=3 imbalance_ratio=2.828476 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[30884, 185367, 81087, 2399, 42281, 6603, 58158, 117509] avg=65536.0000 max=185367 max_rank=1 min=2399 min_rank=3 imbalance_ratio=2.828476 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[30884, 185367, 81087, 2399, 42281, 6603, 58158, 117509] avg=65536.0000 max=185367 max_rank=1 min=2399 min_rank=3 imbalance_ratio=2.828476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119174 max_load_reduction_pct=0.642908 moved_to_replicas=185387 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119174 max_load_reduction_pct=0.642908 moved_to_replicas=185387 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119174 max_load_reduction_pct=0.642908 moved_to_replicas=185387 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119174 max_load_reduction_pct=0.642908 moved_to_replicas=185387 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[30884, 185367, 81087, 2399, 42281, 6603, 58158, 117509] avg=65536.0000 max=185367 max_rank=1 min=2399 min_rank=3 imbalance_ratio=2.828476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[30884, 185367, 81087, 2399, 42281, 6603, 58158, 117509] avg=65536.0000 max=185367 max_rank=1 min=2399 min_rank=3 imbalance_ratio=2.828476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[30884, 185367, 81087, 2399, 42281, 6603, 58158, 117509] avg=65536.0000 max=185367 max_rank=1 min=2399 min_rank=3 imbalance_ratio=2.828476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[30884, 185367, 81087, 2399, 42281, 6603, 58158, 117509] avg=65536.0000 max=185367 max_rank=1 min=2399 min_rank=3 imbalance_ratio=2.828476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65232, 131292, 134, 68633, 16, 140109, 45849, 73023] avg=65536.0000 max=140109 max_rank=5 min=16 min_rank=4 imbalance_ratio=2.137894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65232, 131292, 134, 68633, 16, 140109, 45849, 73023] avg=65536.0000 max=140109 max_rank=5 min=16 min_rank=4 imbalance_ratio=2.137894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65232, 131292, 134, 68633, 16, 140109, 45849, 73023] avg=65536.0000 max=140109 max_rank=5 min=16 min_rank=4 imbalance_ratio=2.137894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119174 max_load_reduction_pct=0.642908 moved_to_replicas=185387 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119174 max_load_reduction_pct=0.642908 moved_to_replicas=185387 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65232, 131292, 134, 68633, 16, 140109, 45849, 73023] avg=65536.0000 max=140109 max_rank=5 min=16 min_rank=4 imbalance_ratio=2.137894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119174 max_load_reduction_pct=0.642908 moved_to_replicas=185387 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[66193, 66191, 66191, 66192, 66192, 66193, 60944, 66192] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=119174 max_load_reduction_pct=0.642908 moved_to_replicas=185387 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] avg=65536.0000 max=66194 max_rank=2 min=62572 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73915 max_load_reduction_pct=0.527554 moved_to_replicas=148289 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] avg=65536.0000 max=66194 max_rank=2 min=62572 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73915 max_load_reduction_pct=0.527554 moved_to_replicas=148289 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] avg=65536.0000 max=66194 max_rank=2 min=62572 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73915 max_load_reduction_pct=0.527554 moved_to_replicas=148289 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] avg=65536.0000 max=66194 max_rank=2 min=62572 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73915 max_load_reduction_pct=0.527554 moved_to_replicas=148289 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65232, 131292, 134, 68633, 16, 140109, 45849, 73023] avg=65536.0000 max=140109 max_rank=5 min=16 min_rank=4 imbalance_ratio=2.137894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65232, 131292, 134, 68633, 16, 140109, 45849, 73023] avg=65536.0000 max=140109 max_rank=5 min=16 min_rank=4 imbalance_ratio=2.137894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65232, 131292, 134, 68633, 16, 140109, 45849, 73023] avg=65536.0000 max=140109 max_rank=5 min=16 min_rank=4 imbalance_ratio=2.137894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65232, 131292, 134, 68633, 16, 140109, 45849, 73023] avg=65536.0000 max=140109 max_rank=5 min=16 min_rank=4 imbalance_ratio=2.137894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65344, 131402, 65503, 65575, 21, 129957, 876, 65610] avg=65536.0000 max=131402 max_rank=1 min=21 min_rank=4 imbalance_ratio=2.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65344, 131402, 65503, 65575, 21, 129957, 876, 65610] avg=65536.0000 max=131402 max_rank=1 min=21 min_rank=4 imbalance_ratio=2.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65344, 131402, 65503, 65575, 21, 129957, 876, 65610] avg=65536.0000 max=131402 max_rank=1 min=21 min_rank=4 imbalance_ratio=2.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65344, 131402, 65503, 65575, 21, 129957, 876, 65610] avg=65536.0000 max=131402 max_rank=1 min=21 min_rank=4 imbalance_ratio=2.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] avg=65536.0000 max=66194 max_rank=2 min=62572 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73915 max_load_reduction_pct=0.527554 moved_to_replicas=148289 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] avg=65536.0000 max=66194 max_rank=2 min=62572 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73915 max_load_reduction_pct=0.527554 moved_to_replicas=148289 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] avg=65536.0000 max=66194 max_rank=2 min=62572 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73915 max_load_reduction_pct=0.527554 moved_to_replicas=148289 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65232, 66192, 66194, 66192, 65524, 66190, 62572, 66192] avg=65536.0000 max=66194 max_rank=2 min=62572 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73915 max_load_reduction_pct=0.527554 moved_to_replicas=148289 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] avg=65536.0000 max=66192 max_rank=1 min=64641 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65210 max_load_reduction_pct=0.496263 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] avg=65536.0000 max=66192 max_rank=1 min=64641 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65210 max_load_reduction_pct=0.496263 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] avg=65536.0000 max=66192 max_rank=1 min=64641 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65210 max_load_reduction_pct=0.496263 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] avg=65536.0000 max=66192 max_rank=1 min=64641 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65210 max_load_reduction_pct=0.496263 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65344, 131402, 65503, 65575, 21, 129957, 876, 65610] avg=65536.0000 max=131402 max_rank=1 min=21 min_rank=4 imbalance_ratio=2.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65344, 131402, 65503, 65575, 21, 129957, 876, 65610] avg=65536.0000 max=131402 max_rank=1 min=21 min_rank=4 imbalance_ratio=2.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65344, 131402, 65503, 65575, 21, 129957, 876, 65610] avg=65536.0000 max=131402 max_rank=1 min=21 min_rank=4 imbalance_ratio=2.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65344, 131402, 65503, 65575, 21, 129957, 876, 65610] avg=65536.0000 max=131402 max_rank=1 min=21 min_rank=4 imbalance_ratio=2.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[64214, 195778, 59433, 7053, 4194, 39, 62262, 131315] avg=65536.0000 max=195778 max_rank=1 min=39 min_rank=5 imbalance_ratio=2.987335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[64214, 195778, 59433, 7053, 4194, 39, 62262, 131315] avg=65536.0000 max=195778 max_rank=1 min=39 min_rank=5 imbalance_ratio=2.987335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[64214, 195778, 59433, 7053, 4194, 39, 62262, 131315] avg=65536.0000 max=195778 max_rank=1 min=39 min_rank=5 imbalance_ratio=2.987335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[64214, 195778, 59433, 7053, 4194, 39, 62262, 131315] avg=65536.0000 max=195778 max_rank=1 min=39 min_rank=5 imbalance_ratio=2.987335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] avg=65536.0000 max=66192 max_rank=1 min=64641 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65210 max_load_reduction_pct=0.496263 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] avg=65536.0000 max=66192 max_rank=1 min=64641 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65210 max_load_reduction_pct=0.496263 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] avg=65536.0000 max=66192 max_rank=1 min=64641 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65210 max_load_reduction_pct=0.496263 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] avg=65536.0000 max=66192 max_rank=1 min=63614 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129586 max_load_reduction_pct=0.661903 moved_to_replicas=194709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65344, 66192, 65503, 65575, 65231, 66192, 64641, 65610] avg=65536.0000 max=66192 max_rank=1 min=64641 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65210 max_load_reduction_pct=0.496263 moved_to_replicas=128975 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] avg=65536.0000 max=66192 max_rank=1 min=63614 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129586 max_load_reduction_pct=0.661903 moved_to_replicas=194709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] avg=65536.0000 max=66192 max_rank=1 min=63614 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129586 max_load_reduction_pct=0.661903 moved_to_replicas=194709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] avg=65536.0000 max=66192 max_rank=1 min=63614 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129586 max_load_reduction_pct=0.661903 moved_to_replicas=194709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[64214, 195778, 59433, 7053, 4194, 39, 62262, 131315] avg=65536.0000 max=195778 max_rank=1 min=39 min_rank=5 imbalance_ratio=2.987335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[64214, 195778, 59433, 7053, 4194, 39, 62262, 131315] avg=65536.0000 max=195778 max_rank=1 min=39 min_rank=5 imbalance_ratio=2.987335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[64214, 195778, 59433, 7053, 4194, 39, 62262, 131315] avg=65536.0000 max=195778 max_rank=1 min=39 min_rank=5 imbalance_ratio=2.987335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[139958, 130944, 132, 65539, 84308, 22, 37855, 65530] avg=65536.0000 max=139958 max_rank=0 min=22 min_rank=5 imbalance_ratio=2.135590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[139958, 130944, 132, 65539, 84308, 22, 37855, 65530] avg=65536.0000 max=139958 max_rank=0 min=22 min_rank=5 imbalance_ratio=2.135590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[64214, 195778, 59433, 7053, 4194, 39, 62262, 131315] avg=65536.0000 max=195778 max_rank=1 min=39 min_rank=5 imbalance_ratio=2.987335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[139958, 130944, 132, 65539, 84308, 22, 37855, 65530] avg=65536.0000 max=139958 max_rank=0 min=22 min_rank=5 imbalance_ratio=2.135590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[139958, 130944, 132, 65539, 84308, 22, 37855, 65530] avg=65536.0000 max=139958 max_rank=0 min=22 min_rank=5 imbalance_ratio=2.135590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] avg=65536.0000 max=66192 max_rank=1 min=63614 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129586 max_load_reduction_pct=0.661903 moved_to_replicas=194709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] avg=65536.0000 max=66192 max_rank=1 min=63614 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129586 max_load_reduction_pct=0.661903 moved_to_replicas=194709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] avg=65536.0000 max=66192 max_rank=1 min=63614 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129586 max_load_reduction_pct=0.661903 moved_to_replicas=194709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] avg=65536.0000 max=66194 max_rank=0 min=62994 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73764 max_load_reduction_pct=0.527044 moved_to_replicas=156634 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] avg=65536.0000 max=66194 max_rank=0 min=62994 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73764 max_load_reduction_pct=0.527044 moved_to_replicas=156634 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] avg=65536.0000 max=66194 max_rank=0 min=62994 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73764 max_load_reduction_pct=0.527044 moved_to_replicas=156634 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[64214, 66192, 66192, 66192, 66192, 65500, 63614, 66192] avg=65536.0000 max=66192 max_rank=1 min=63614 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=129586 max_load_reduction_pct=0.661903 moved_to_replicas=194709 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] avg=65536.0000 max=66194 max_rank=0 min=62994 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73764 max_load_reduction_pct=0.527044 moved_to_replicas=156634 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[139958, 130944, 132, 65539, 84308, 22, 37855, 65530] avg=65536.0000 max=139958 max_rank=0 min=22 min_rank=5 imbalance_ratio=2.135590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[139958, 130944, 132, 65539, 84308, 22, 37855, 65530] avg=65536.0000 max=139958 max_rank=0 min=22 min_rank=5 imbalance_ratio=2.135590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[130637, 130776, 129875, 47781, 65539, 369, 19256, 55] avg=65536.0000 max=130776 max_rank=1 min=55 min_rank=7 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[130637, 130776, 129875, 47781, 65539, 369, 19256, 55] avg=65536.0000 max=130776 max_rank=1 min=55 min_rank=7 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[139958, 130944, 132, 65539, 84308, 22, 37855, 65530] avg=65536.0000 max=139958 max_rank=0 min=22 min_rank=5 imbalance_ratio=2.135590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[130637, 130776, 129875, 47781, 65539, 369, 19256, 55] avg=65536.0000 max=130776 max_rank=1 min=55 min_rank=7 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[139958, 130944, 132, 65539, 84308, 22, 37855, 65530] avg=65536.0000 max=139958 max_rank=0 min=22 min_rank=5 imbalance_ratio=2.135590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[130637, 130776, 129875, 47781, 65539, 369, 19256, 55] avg=65536.0000 max=130776 max_rank=1 min=55 min_rank=7 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] avg=65536.0000 max=66194 max_rank=0 min=62994 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73764 max_load_reduction_pct=0.527044 moved_to_replicas=156634 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] avg=65536.0000 max=66194 max_rank=0 min=62994 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73764 max_load_reduction_pct=0.527044 moved_to_replicas=156634 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] avg=65536.0000 max=66192 max_rank=0 min=64529 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64584 max_load_reduction_pct=0.493852 moved_to_replicas=192712 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] avg=65536.0000 max=66192 max_rank=0 min=64529 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64584 max_load_reduction_pct=0.493852 moved_to_replicas=192712 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] avg=65536.0000 max=66192 max_rank=0 min=64529 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64584 max_load_reduction_pct=0.493852 moved_to_replicas=192712 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] avg=65536.0000 max=66194 max_rank=0 min=62994 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73764 max_load_reduction_pct=0.527044 moved_to_replicas=156634 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[66194, 66192, 66190, 65539, 66191, 65458, 62994, 65530] avg=65536.0000 max=66194 max_rank=0 min=62994 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=73764 max_load_reduction_pct=0.527044 moved_to_replicas=156634 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] avg=65536.0000 max=66192 max_rank=0 min=64529 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64584 max_load_reduction_pct=0.493852 moved_to_replicas=192712 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[130637, 130776, 129875, 47781, 65539, 369, 19256, 55] avg=65536.0000 max=130776 max_rank=1 min=55 min_rank=7 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[130637, 130776, 129875, 47781, 65539, 369, 19256, 55] avg=65536.0000 max=130776 max_rank=1 min=55 min_rank=7 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63388, 124049, 72, 140106, 65645, 65373, 125] avg=65536.0000 max=140106 max_rank=4 min=72 min_rank=3 imbalance_ratio=2.137848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63388, 124049, 72, 140106, 65645, 65373, 125] avg=65536.0000 max=140106 max_rank=4 min=72 min_rank=3 imbalance_ratio=2.137848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63388, 124049, 72, 140106, 65645, 65373, 125] avg=65536.0000 max=140106 max_rank=4 min=72 min_rank=3 imbalance_ratio=2.137848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[130637, 130776, 129875, 47781, 65539, 369, 19256, 55] avg=65536.0000 max=130776 max_rank=1 min=55 min_rank=7 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[130637, 130776, 129875, 47781, 65539, 369, 19256, 55] avg=65536.0000 max=130776 max_rank=1 min=55 min_rank=7 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63388, 124049, 72, 140106, 65645, 65373, 125] avg=65536.0000 max=140106 max_rank=4 min=72 min_rank=3 imbalance_ratio=2.137848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] avg=65536.0000 max=66192 max_rank=0 min=64529 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64584 max_load_reduction_pct=0.493852 moved_to_replicas=192712 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] avg=65536.0000 max=66192 max_rank=0 min=64529 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64584 max_load_reduction_pct=0.493852 moved_to_replicas=192712 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] avg=65536.0000 max=66193 max_rank=7 min=63596 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=73913 max_load_reduction_pct=0.527551 moved_to_replicas=131771 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] avg=65536.0000 max=66193 max_rank=7 min=63596 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=73913 max_load_reduction_pct=0.527551 moved_to_replicas=131771 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] avg=65536.0000 max=66193 max_rank=7 min=63596 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=73913 max_load_reduction_pct=0.527551 moved_to_replicas=131771 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] avg=65536.0000 max=66192 max_rank=0 min=64529 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64584 max_load_reduction_pct=0.493852 moved_to_replicas=192712 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[66192, 66192, 66192, 64529, 65539, 64814, 66191, 64639] avg=65536.0000 max=66192 max_rank=0 min=64529 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64584 max_load_reduction_pct=0.493852 moved_to_replicas=192712 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] avg=65536.0000 max=66193 max_rank=7 min=63596 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=73913 max_load_reduction_pct=0.527551 moved_to_replicas=131771 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[2033, 65821, 55964, 107, 27, 195474, 63016, 141846] avg=65536.0000 max=195474 max_rank=5 min=27 min_rank=4 imbalance_ratio=2.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63388, 124049, 72, 140106, 65645, 65373, 125] avg=65536.0000 max=140106 max_rank=4 min=72 min_rank=3 imbalance_ratio=2.137848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63388, 124049, 72, 140106, 65645, 65373, 125] avg=65536.0000 max=140106 max_rank=4 min=72 min_rank=3 imbalance_ratio=2.137848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[2033, 65821, 55964, 107, 27, 195474, 63016, 141846] avg=65536.0000 max=195474 max_rank=5 min=27 min_rank=4 imbalance_ratio=2.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[2033, 65821, 55964, 107, 27, 195474, 63016, 141846] avg=65536.0000 max=195474 max_rank=5 min=27 min_rank=4 imbalance_ratio=2.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63388, 124049, 72, 140106, 65645, 65373, 125] avg=65536.0000 max=140106 max_rank=4 min=72 min_rank=3 imbalance_ratio=2.137848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63388, 124049, 72, 140106, 65645, 65373, 125] avg=65536.0000 max=140106 max_rank=4 min=72 min_rank=3 imbalance_ratio=2.137848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[2033, 65821, 55964, 107, 27, 195474, 63016, 141846] avg=65536.0000 max=195474 max_rank=5 min=27 min_rank=4 imbalance_ratio=2.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] avg=65536.0000 max=66194 max_rank=2 min=63881 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=129280 max_load_reduction_pct=0.661367 moved_to_replicas=204936 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] avg=65536.0000 max=66193 max_rank=7 min=63596 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=73913 max_load_reduction_pct=0.527551 moved_to_replicas=131771 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] avg=65536.0000 max=66193 max_rank=7 min=63596 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=73913 max_load_reduction_pct=0.527551 moved_to_replicas=131771 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] avg=65536.0000 max=66194 max_rank=2 min=63881 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=129280 max_load_reduction_pct=0.661367 moved_to_replicas=204936 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] avg=65536.0000 max=66194 max_rank=2 min=63881 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=129280 max_load_reduction_pct=0.661367 moved_to_replicas=204936 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] avg=65536.0000 max=66193 max_rank=7 min=63596 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=73913 max_load_reduction_pct=0.527551 moved_to_replicas=131771 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65530, 63596, 66192, 65568, 66191, 65645, 65373, 66193] avg=65536.0000 max=66193 max_rank=7 min=63596 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=73913 max_load_reduction_pct=0.527551 moved_to_replicas=131771 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] avg=65536.0000 max=66194 max_rank=2 min=63881 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=129280 max_load_reduction_pct=0.661367 moved_to_replicas=204936 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[130070, 65, 65481, 28515, 103373, 64682, 66626, 65476] avg=65536.0000 max=130070 max_rank=0 min=65 min_rank=1 imbalance_ratio=1.984711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[2033, 65821, 55964, 107, 27, 195474, 63016, 141846] avg=65536.0000 max=195474 max_rank=5 min=27 min_rank=4 imbalance_ratio=2.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[2033, 65821, 55964, 107, 27, 195474, 63016, 141846] avg=65536.0000 max=195474 max_rank=5 min=27 min_rank=4 imbalance_ratio=2.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[130070, 65, 65481, 28515, 103373, 64682, 66626, 65476] avg=65536.0000 max=130070 max_rank=0 min=65 min_rank=1 imbalance_ratio=1.984711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[130070, 65, 65481, 28515, 103373, 64682, 66626, 65476] avg=65536.0000 max=130070 max_rank=0 min=65 min_rank=1 imbalance_ratio=1.984711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[2033, 65821, 55964, 107, 27, 195474, 63016, 141846] avg=65536.0000 max=195474 max_rank=5 min=27 min_rank=4 imbalance_ratio=2.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] avg=65536.0000 max=66192 max_rank=0 min=64377 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63878 max_load_reduction_pct=0.491105 moved_to_replicas=101493 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[2033, 65821, 55964, 107, 27, 195474, 63016, 141846] avg=65536.0000 max=195474 max_rank=5 min=27 min_rank=4 imbalance_ratio=2.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[130070, 65, 65481, 28515, 103373, 64682, 66626, 65476] avg=65536.0000 max=130070 max_rank=0 min=65 min_rank=1 imbalance_ratio=1.984711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] avg=65536.0000 max=66194 max_rank=2 min=63881 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=129280 max_load_reduction_pct=0.661367 moved_to_replicas=204936 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] avg=65536.0000 max=66194 max_rank=2 min=63881 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=129280 max_load_reduction_pct=0.661367 moved_to_replicas=204936 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] avg=65536.0000 max=66192 max_rank=0 min=64377 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63878 max_load_reduction_pct=0.491105 moved_to_replicas=101493 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] avg=65536.0000 max=66192 max_rank=0 min=64377 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63878 max_load_reduction_pct=0.491105 moved_to_replicas=101493 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] avg=65536.0000 max=66194 max_rank=2 min=63881 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=129280 max_load_reduction_pct=0.661367 moved_to_replicas=204936 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65821, 66194, 63881, 65535, 66192, 64283, 66190] avg=65536.0000 max=66194 max_rank=2 min=63881 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=129280 max_load_reduction_pct=0.661367 moved_to_replicas=204936 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] avg=65536.0000 max=66192 max_rank=0 min=64377 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63878 max_load_reduction_pct=0.491105 moved_to_replicas=101493 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[63212, 65772, 65344, 65539, 130362, 67901, 63671, 2487] avg=65536.0000 max=130362 max_rank=4 min=2487 min_rank=7 imbalance_ratio=1.989166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[130070, 65, 65481, 28515, 103373, 64682, 66626, 65476] avg=65536.0000 max=130070 max_rank=0 min=65 min_rank=1 imbalance_ratio=1.984711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[130070, 65, 65481, 28515, 103373, 64682, 66626, 65476] avg=65536.0000 max=130070 max_rank=0 min=65 min_rank=1 imbalance_ratio=1.984711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[63212, 65772, 65344, 65539, 130362, 67901, 63671, 2487] avg=65536.0000 max=130362 max_rank=4 min=2487 min_rank=7 imbalance_ratio=1.989166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[63212, 65772, 65344, 65539, 130362, 67901, 63671, 2487] avg=65536.0000 max=130362 max_rank=4 min=2487 min_rank=7 imbalance_ratio=1.989166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] avg=65536.0000 max=66192 max_rank=4 min=63677 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64170 max_load_reduction_pct=0.492245 moved_to_replicas=65879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[130070, 65, 65481, 28515, 103373, 64682, 66626, 65476] avg=65536.0000 max=130070 max_rank=0 min=65 min_rank=1 imbalance_ratio=1.984711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[130070, 65, 65481, 28515, 103373, 64682, 66626, 65476] avg=65536.0000 max=130070 max_rank=0 min=65 min_rank=1 imbalance_ratio=1.984711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[63212, 65772, 65344, 65539, 130362, 67901, 63671, 2487] avg=65536.0000 max=130362 max_rank=4 min=2487 min_rank=7 imbalance_ratio=1.989166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] avg=65536.0000 max=66192 max_rank=0 min=64377 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63878 max_load_reduction_pct=0.491105 moved_to_replicas=101493 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] avg=65536.0000 max=66192 max_rank=0 min=64377 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63878 max_load_reduction_pct=0.491105 moved_to_replicas=101493 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] avg=65536.0000 max=66192 max_rank=4 min=63677 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64170 max_load_reduction_pct=0.492245 moved_to_replicas=65879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] avg=65536.0000 max=66192 max_rank=4 min=63677 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64170 max_load_reduction_pct=0.492245 moved_to_replicas=65879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] avg=65536.0000 max=66192 max_rank=0 min=64377 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63878 max_load_reduction_pct=0.491105 moved_to_replicas=101493 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64377, 65481, 65697, 66191, 64682, 66192, 65476] avg=65536.0000 max=66192 max_rank=0 min=64377 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63878 max_load_reduction_pct=0.491105 moved_to_replicas=101493 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] avg=65536.0000 max=66192 max_rank=4 min=63677 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64170 max_load_reduction_pct=0.492245 moved_to_replicas=65879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65365, 75492, 56579, 195933, 65, 65253, 65582, 19] avg=65536.0000 max=195933 max_rank=3 min=19 min_rank=7 imbalance_ratio=2.989700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[63212, 65772, 65344, 65539, 130362, 67901, 63671, 2487] avg=65536.0000 max=130362 max_rank=4 min=2487 min_rank=7 imbalance_ratio=1.989166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[63212, 65772, 65344, 65539, 130362, 67901, 63671, 2487] avg=65536.0000 max=130362 max_rank=4 min=2487 min_rank=7 imbalance_ratio=1.989166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65365, 75492, 56579, 195933, 65, 65253, 65582, 19] avg=65536.0000 max=195933 max_rank=3 min=19 min_rank=7 imbalance_ratio=2.989700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] avg=65536.0000 max=66192 max_rank=1 min=64312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129741 max_load_reduction_pct=0.662170 moved_to_replicas=139041 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65365, 75492, 56579, 195933, 65, 65253, 65582, 19] avg=65536.0000 max=195933 max_rank=3 min=19 min_rank=7 imbalance_ratio=2.989700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[63212, 65772, 65344, 65539, 130362, 67901, 63671, 2487] avg=65536.0000 max=130362 max_rank=4 min=2487 min_rank=7 imbalance_ratio=1.989166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[63212, 65772, 65344, 65539, 130362, 67901, 63671, 2487] avg=65536.0000 max=130362 max_rank=4 min=2487 min_rank=7 imbalance_ratio=1.989166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65365, 75492, 56579, 195933, 65, 65253, 65582, 19] avg=65536.0000 max=195933 max_rank=3 min=19 min_rank=7 imbalance_ratio=2.989700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] avg=65536.0000 max=66192 max_rank=4 min=63677 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64170 max_load_reduction_pct=0.492245 moved_to_replicas=65879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] avg=65536.0000 max=66192 max_rank=4 min=63677 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64170 max_load_reduction_pct=0.492245 moved_to_replicas=65879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] avg=65536.0000 max=66192 max_rank=1 min=64312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129741 max_load_reduction_pct=0.662170 moved_to_replicas=139041 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] avg=65536.0000 max=66192 max_rank=1 min=64312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129741 max_load_reduction_pct=0.662170 moved_to_replicas=139041 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] avg=65536.0000 max=66192 max_rank=4 min=63677 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64170 max_load_reduction_pct=0.492245 moved_to_replicas=65879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[66176, 397, 130110, 638, 130405, 65681, 65345, 65536] avg=65536.0000 max=130405 max_rank=4 min=397 min_rank=1 imbalance_ratio=1.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[63677, 65772, 65344, 65539, 66192, 66192, 65380, 66192] avg=65536.0000 max=66192 max_rank=4 min=63677 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64170 max_load_reduction_pct=0.492245 moved_to_replicas=65879 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] avg=65536.0000 max=66192 max_rank=1 min=64312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129741 max_load_reduction_pct=0.662170 moved_to_replicas=139041 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[66176, 397, 130110, 638, 130405, 65681, 65345, 65536] avg=65536.0000 max=130405 max_rank=4 min=397 min_rank=1 imbalance_ratio=1.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] avg=65536.0000 max=66192 max_rank=2 min=64556 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64213 max_load_reduction_pct=0.492412 moved_to_replicas=128131 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65365, 75492, 56579, 195933, 65, 65253, 65582, 19] avg=65536.0000 max=195933 max_rank=3 min=19 min_rank=7 imbalance_ratio=2.989700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65365, 75492, 56579, 195933, 65, 65253, 65582, 19] avg=65536.0000 max=195933 max_rank=3 min=19 min_rank=7 imbalance_ratio=2.989700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[66176, 397, 130110, 638, 130405, 65681, 65345, 65536] avg=65536.0000 max=130405 max_rank=4 min=397 min_rank=1 imbalance_ratio=1.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65365, 75492, 56579, 195933, 65, 65253, 65582, 19] avg=65536.0000 max=195933 max_rank=3 min=19 min_rank=7 imbalance_ratio=2.989700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65365, 75492, 56579, 195933, 65, 65253, 65582, 19] avg=65536.0000 max=195933 max_rank=3 min=19 min_rank=7 imbalance_ratio=2.989700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[66176, 397, 130110, 638, 130405, 65681, 65345, 65536] avg=65536.0000 max=130405 max_rank=4 min=397 min_rank=1 imbalance_ratio=1.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] avg=65536.0000 max=66192 max_rank=2 min=64556 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64213 max_load_reduction_pct=0.492412 moved_to_replicas=128131 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] avg=65536.0000 max=66192 max_rank=1 min=64312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129741 max_load_reduction_pct=0.662170 moved_to_replicas=139041 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] avg=65536.0000 max=66192 max_rank=1 min=64312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129741 max_load_reduction_pct=0.662170 moved_to_replicas=139041 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] avg=65536.0000 max=66192 max_rank=2 min=64556 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64213 max_load_reduction_pct=0.492412 moved_to_replicas=128131 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] avg=65536.0000 max=66192 max_rank=1 min=64312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129741 max_load_reduction_pct=0.662170 moved_to_replicas=139041 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65365, 66192, 65879, 66192, 64312, 65253, 65582, 65513] avg=65536.0000 max=66192 max_rank=1 min=64312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129741 max_load_reduction_pct=0.662170 moved_to_replicas=139041 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] avg=65536.0000 max=66192 max_rank=2 min=64556 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64213 max_load_reduction_pct=0.492412 moved_to_replicas=128131 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[66176, 397, 130110, 638, 130405, 65681, 65345, 65536] avg=65536.0000 max=130405 max_rank=4 min=397 min_rank=1 imbalance_ratio=1.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[66176, 397, 130110, 638, 130405, 65681, 65345, 65536] avg=65536.0000 max=130405 max_rank=4 min=397 min_rank=1 imbalance_ratio=1.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[66176, 397, 130110, 638, 130405, 65681, 65345, 65536] avg=65536.0000 max=130405 max_rank=4 min=397 min_rank=1 imbalance_ratio=1.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[66176, 397, 130110, 638, 130405, 65681, 65345, 65536] avg=65536.0000 max=130405 max_rank=4 min=397 min_rank=1 imbalance_ratio=1.989822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] avg=65536.0000 max=66192 max_rank=2 min=64556 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64213 max_load_reduction_pct=0.492412 moved_to_replicas=128131 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] avg=65536.0000 max=66192 max_rank=2 min=64556 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64213 max_load_reduction_pct=0.492412 moved_to_replicas=128131 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] avg=65536.0000 max=66192 max_rank=2 min=64556 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64213 max_load_reduction_pct=0.492412 moved_to_replicas=128131 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[66176, 64610, 66192, 64556, 66192, 65681, 65345, 65536] avg=65536.0000 max=66192 max_rank=2 min=64556 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64213 max_load_reduction_pct=0.492412 moved_to_replicas=128131 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=12 total_expected_recv=791744 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=180096 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=12 total_expected_recv=773111 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=200064 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=12 total_expected_recv=787992 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183680 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=12 total_expected_recv=789482 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183424 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=12 total_expected_recv=786501 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187008 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=12 total_expected_recv=789455 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182656 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=12 total_expected_recv=787519 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=12 total_expected_recv=785652 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 15:44:40] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 341.8 | throughput per GPU (TFLOP/s/GPU): 529.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.051292E+01 | loss scale: 1.0 | grad norm: 95.357 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=81920 expected_recv=61816 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65319 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=81920 expected_recv=61816 actual_valid_recv=63488 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=81920 expected_recv=62846 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=81920 expected_recv=64290 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65319 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=81920 expected_recv=64716 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=81920 expected_recv=63725 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=81920 expected_recv=64716 actual_valid_recv=65536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=81920 expected_recv=63690 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=81920 expected_recv=64750 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=81920 expected_recv=63725 actual_valid_recv=65024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=81920 expected_recv=63690 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=81920 expected_recv=64750 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=81920 expected_recv=63259 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=81920 expected_recv=63245 expected_overflow=0 exact_recv_per_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65596 expected_overflow=0 exact_recv_per_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65542 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=81920 expected_recv=63259 actual_valid_recv=63616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65595 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=81920 expected_recv=63245 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65136 expected_overflow=0 exact_recv_per_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65596 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65122 expected_overflow=0 exact_recv_per_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65595 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65122 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65136 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64749 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65520 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64749 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64406 expected_overflow=0 exact_recv_per_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64626 expected_overflow=0 exact_recv_per_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64626 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65518 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64481 expected_overflow=0 exact_recv_per_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64406 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64067 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64067 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65490 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64930 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=81920 expected_recv=64481 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65668 expected_overflow=0 exact_recv_per_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65668 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65490 expected_overflow=0 exact_recv_per_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64119 expected_overflow=0 exact_recv_per_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64933 expected_overflow=0 exact_recv_per_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65490 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64119 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64933 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65696 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=81920 expected_recv=64930 actual_valid_recv=65408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65503 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65696 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65985 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64696 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65985 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64696 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65292 expected_overflow=0 exact_recv_per_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65292 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65931 expected_overflow=0 exact_recv_per_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64823 expected_overflow=0 exact_recv_per_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64819 expected_overflow=0 exact_recv_per_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65931 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64823 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64832 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64819 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=81920 expected_recv=64832 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65351 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=65920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65351 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65512 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65512 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65569 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65508 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65568 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=81920 expected_recv=63592 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=65920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=81920 expected_recv=62307 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=81920 expected_recv=63592 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65893 expected_overflow=0 exact_recv_per_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=81920 expected_recv=62307 actual_valid_recv=62976 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65893 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=81920 expected_recv=63591 expected_overflow=0 exact_recv_per_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65495 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=81920 expected_recv=63591 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65522 expected_overflow=0 exact_recv_per_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65357 expected_overflow=0 exact_recv_per_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65522 actual_valid_recv=65920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65632 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65632 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65357 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65641 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64856 expected_overflow=0 exact_recv_per_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64930 expected_overflow=0 exact_recv_per_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=81920 expected_recv=62567 expected_overflow=0 exact_recv_per_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64862 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64930 actual_valid_recv=65280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64862 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64931 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64931 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=81920 expected_recv=64856 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=81920 expected_recv=62567 actual_valid_recv=63232 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65201 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65370 expected_overflow=0 exact_recv_per_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65201 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65370 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=63232 slack_slots=18688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65238 expected_overflow=0 exact_recv_per_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65238 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65140 expected_overflow=0 exact_recv_per_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65163 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65163 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65561 expected_overflow=0 exact_recv_per_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65566 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65140 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65561 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65598 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65495 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65244 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65490 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65490 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=81920 expected_recv=63983 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65397 expected_overflow=0 exact_recv_per_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65410 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=81920 expected_recv=63983 actual_valid_recv=64768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65244 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=81920 expected_recv=64189 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65579 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65397 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65410 actual_valid_recv=65792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=81920 expected_recv=64189 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65269 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65261 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65269 actual_valid_recv=65792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65261 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=81920 expected_recv=64567 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=81920 expected_recv=64567 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=81920 expected_recv=64962 expected_overflow=0 exact_recv_per_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=81920 expected_recv=64962 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65314 expected_overflow=0 exact_recv_per_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=81920 expected_recv=64978 expected_overflow=0 exact_recv_per_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65314 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=81920 expected_recv=64978 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[111920, 159752, 76949, 8874, 36769, 12773, 50195, 67056] avg=65536.0000 max=159752 max_rank=1 min=8874 min_rank=3 imbalance_ratio=2.437622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65496 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65517 actual_valid_recv=65920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65629 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65629 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[111920, 159752, 76949, 8874, 36769, 12773, 50195, 67056] avg=65536.0000 max=159752 max_rank=1 min=8874 min_rank=3 imbalance_ratio=2.437622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[111920, 159752, 76949, 8874, 36769, 12773, 50195, 67056] avg=65536.0000 max=159752 max_rank=1 min=8874 min_rank=3 imbalance_ratio=2.437622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] avg=65536.0000 max=66193 max_rank=0 min=61816 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93559 max_load_reduction_pct=0.585652 moved_to_replicas=150909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[111920, 159752, 76949, 8874, 36769, 12773, 50195, 67056] avg=65536.0000 max=159752 max_rank=1 min=8874 min_rank=3 imbalance_ratio=2.437622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] avg=65536.0000 max=66193 max_rank=0 min=61816 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93559 max_load_reduction_pct=0.585652 moved_to_replicas=150909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[111920, 159752, 76949, 8874, 36769, 12773, 50195, 67056] avg=65536.0000 max=159752 max_rank=1 min=8874 min_rank=3 imbalance_ratio=2.437622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] avg=65536.0000 max=66193 max_rank=0 min=61816 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93559 max_load_reduction_pct=0.585652 moved_to_replicas=150909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[111920, 159752, 76949, 8874, 36769, 12773, 50195, 67056] avg=65536.0000 max=159752 max_rank=1 min=8874 min_rank=3 imbalance_ratio=2.437622 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65399 expected_overflow=0 exact_recv_per_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65399 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[111446, 159966, 77809, 8116, 36958, 12573, 48795, 68625] avg=65536.0000 max=159966 max_rank=1 min=8116 min_rank=3 imbalance_ratio=2.440887 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[66192, 66192, 66191, 66192, 66193, 64290, 62846, 66192] avg=65536.0000 max=66193 max_rank=4 min=62846 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93773 max_load_reduction_pct=0.586206 moved_to_replicas=153078 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[126892, 196376, 66581, 67473, 16, 91, 65539, 1320] avg=65536.0000 max=196376 max_rank=1 min=16 min_rank=4 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65541, 64750, 65539, 63690] avg=65536.0000 max=66192 max_rank=0 min=63690 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130184 max_load_reduction_pct=0.662932 moved_to_replicas=192554 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[137, 131021, 56812, 131177, 34, 65595, 34, 139478] avg=65536.0000 max=139478 max_rank=7 min=34 min_rank=4 imbalance_ratio=2.128265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[65122, 66192, 63259, 66192, 65544, 65595, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63259 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=73286 max_load_reduction_pct=0.525431 moved_to_replicas=203100 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[83, 86890, 28, 60301, 49942, 69, 65544, 261431] avg=65536.0000 max=261431 max_rank=7 min=28 min_rank=2 imbalance_ratio=3.989120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[64406, 66190, 65520, 64749, 66194, 65493, 65544, 66192] avg=65536.0000 max=66194 max_rank=4 min=64406 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=195237 max_load_reduction_pct=0.746801 moved_to_replicas=215937 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65530, 96205, 62, 65490, 100183, 27, 65696, 131095] avg=65536.0000 max=131095 max_rank=7 min=27 min_rank=5 imbalance_ratio=2.000351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65530, 66192, 64067, 65490, 66191, 64930, 65696, 66192] avg=65536.0000 max=66192 max_rank=1 min=64067 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64903 max_load_reduction_pct=0.495084 moved_to_replicas=128907 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[196414, 65985, 188, 130700, 65501, 110, 65351, 39] avg=65536.0000 max=196414 max_rank=0 min=39 min_rank=7 imbalance_ratio=2.997040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65985, 64696, 66192, 65501, 64832, 65351, 65539] avg=65536.0000 max=66192 max_rank=0 min=64696 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130222 max_load_reduction_pct=0.662998 moved_to_replicas=194730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[121114, 65512, 235380, 109, 813, 65511, 39, 35810] avg=65536.0000 max=235380 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.591614 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66191, 65568, 63592, 65511, 65531, 66191] avg=65536.0000 max=66192 max_rank=0 min=63592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=169188 max_load_reduction_pct=0.718787 moved_to_replicas=224110 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[26, 131514, 48, 173, 52034, 186063, 23722, 130708] avg=65536.0000 max=186063 max_rank=5 min=26 min_rank=0 imbalance_ratio=2.839096 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65524, 66192, 62307, 65495, 66191, 66194, 66193, 66192] avg=65536.0000 max=66194 max_rank=5 min=62307 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=119869 max_load_reduction_pct=0.644239 moved_to_replicas=249709 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[130914, 65632, 131057, 140, 65554, 66, 65370, 65555] avg=65536.0000 max=131057 max_rank=2 min=66 min_rank=5 imbalance_ratio=1.999771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65632, 66192, 64862, 65554, 64931, 65370, 65555] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64865 max_load_reduction_pct=0.494937 moved_to_replicas=129587 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65558, 103, 131290, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=131290 max_rank=2 min=103 min_rank=1 imbalance_ratio=2.003326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65558, 65201, 66192, 65511, 65598, 65529, 65536, 65163] avg=65536.0000 max=66192 max_rank=2 min=65163 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65098 max_load_reduction_pct=0.495834 moved_to_replicas=65098 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[22, 131226, 65495, 195749, 329, 125, 131132, 210] avg=65536.0000 max=195749 max_rank=3 min=22 min_rank=0 imbalance_ratio=2.986893 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65495, 66192, 65269, 64189, 66192, 65244] avg=65536.0000 max=66192 max_rank=1 min=64189 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129557 max_load_reduction_pct=0.661853 moved_to_replicas=259531 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65629, 1392, 260907, 62527, 65555, 68232, 32, 14] avg=65536.0000 max=260907 max_rank=2 min=14 min_rank=7 imbalance_ratio=3.981125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65629, 65261, 66192, 64567, 65555, 66192, 65399, 65493] avg=65536.0000 max=66192 max_rank=2 min=64567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=194715 max_load_reduction_pct=0.746300 moved_to_replicas=196755 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=12 total_expected_recv=790933 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184192 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=12 total_expected_recv=782494 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192128 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=12 total_expected_recv=787202 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186112 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=12 total_expected_recv=786923 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187904 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=12 total_expected_recv=782436 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191872 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=12 total_expected_recv=787835 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185344 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=12 total_expected_recv=788244 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186624 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=12 total_expected_recv=785389 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188800 peak_demand=66193 required_capacity=1.010025 - [2026-08-26 14:59:19] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 340.6 | throughput per GPU (TFLOP/s/GPU): 530.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.028087E+01 | loss scale: 1.0 | grad norm: 41.959 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65100 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=81920 expected_recv=62777 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65100 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67968 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65451 expected_overflow=0 exact_recv_per_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=81920 expected_recv=62777 actual_valid_recv=64128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67968 slack_slots=13952 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[111920, 159752, 76949, 8874, 36769, 12773, 50195, 67056] avg=65536.0000 max=159752 max_rank=1 min=8874 min_rank=3 imbalance_ratio=2.437622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[127078, 196354, 66583, 67031, 15, 79, 65539, 1609] avg=65536.0000 max=196354 max_rank=1 min=15 min_rank=4 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] avg=65536.0000 max=66193 max_rank=0 min=61816 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93559 max_load_reduction_pct=0.585652 moved_to_replicas=150909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] avg=65536.0000 max=66193 max_rank=0 min=61816 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93559 max_load_reduction_pct=0.585652 moved_to_replicas=150909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[127078, 196354, 66583, 67031, 15, 79, 65539, 1609] avg=65536.0000 max=196354 max_rank=1 min=15 min_rank=4 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] avg=65536.0000 max=66193 max_rank=0 min=61816 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93559 max_load_reduction_pct=0.585652 moved_to_replicas=150909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] avg=65536.0000 max=66193 max_rank=0 min=61816 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93559 max_load_reduction_pct=0.585652 moved_to_replicas=150909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] avg=65536.0000 max=66192 max_rank=0 min=63725 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130162 max_load_reduction_pct=0.662895 moved_to_replicas=192278 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[127078, 196354, 66583, 67031, 15, 79, 65539, 1609] avg=65536.0000 max=196354 max_rank=1 min=15 min_rank=4 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[111920, 159752, 76949, 8874, 36769, 12773, 50195, 67056] avg=65536.0000 max=159752 max_rank=1 min=8874 min_rank=3 imbalance_ratio=2.437622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] avg=65536.0000 max=66192 max_rank=0 min=63725 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130162 max_load_reduction_pct=0.662895 moved_to_replicas=192278 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[127078, 196354, 66583, 67031, 15, 79, 65539, 1609] avg=65536.0000 max=196354 max_rank=1 min=15 min_rank=4 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[127078, 196354, 66583, 67031, 15, 79, 65539, 1609] avg=65536.0000 max=196354 max_rank=1 min=15 min_rank=4 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] avg=65536.0000 max=66192 max_rank=0 min=63725 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130162 max_load_reduction_pct=0.662895 moved_to_replicas=192278 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[127078, 196354, 66583, 67031, 15, 79, 65539, 1609] avg=65536.0000 max=196354 max_rank=1 min=15 min_rank=4 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[143, 131021, 47530, 131185, 37, 65596, 32, 148744] avg=65536.0000 max=148744 max_rank=7 min=32 min_rank=6 imbalance_ratio=2.269653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[127078, 196354, 66583, 67031, 15, 79, 65539, 1609] avg=65536.0000 max=196354 max_rank=1 min=15 min_rank=4 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[66193, 66192, 66192, 66192, 66192, 65319, 61816, 66192] avg=65536.0000 max=66193 max_rank=0 min=61816 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=93559 max_load_reduction_pct=0.585652 moved_to_replicas=150909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] avg=65536.0000 max=66192 max_rank=0 min=63725 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130162 max_load_reduction_pct=0.662895 moved_to_replicas=192278 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] avg=65536.0000 max=66192 max_rank=0 min=63725 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130162 max_load_reduction_pct=0.662895 moved_to_replicas=192278 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[143, 131021, 47530, 131185, 37, 65596, 32, 148744] avg=65536.0000 max=148744 max_rank=7 min=32 min_rank=6 imbalance_ratio=2.269653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] avg=65536.0000 max=66192 max_rank=0 min=63725 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130162 max_load_reduction_pct=0.662895 moved_to_replicas=192278 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] avg=65536.0000 max=66193 max_rank=4 min=63245 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=82551 max_load_reduction_pct=0.554987 moved_to_replicas=212374 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] avg=65536.0000 max=66192 max_rank=0 min=63725 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130162 max_load_reduction_pct=0.662895 moved_to_replicas=192278 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[143, 131021, 47530, 131185, 37, 65596, 32, 148744] avg=65536.0000 max=148744 max_rank=7 min=32 min_rank=6 imbalance_ratio=2.269653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[127078, 196354, 66583, 67031, 15, 79, 65539, 1609] avg=65536.0000 max=196354 max_rank=1 min=15 min_rank=4 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[143, 131021, 47530, 131185, 37, 65596, 32, 148744] avg=65536.0000 max=148744 max_rank=7 min=32 min_rank=6 imbalance_ratio=2.269653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] avg=65536.0000 max=66193 max_rank=4 min=63245 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=82551 max_load_reduction_pct=0.554987 moved_to_replicas=212374 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[143, 131021, 47530, 131185, 37, 65596, 32, 148744] avg=65536.0000 max=148744 max_rank=7 min=32 min_rank=6 imbalance_ratio=2.269653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] avg=65536.0000 max=66193 max_rank=4 min=63245 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=82551 max_load_reduction_pct=0.554987 moved_to_replicas=212374 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[79, 114431, 25, 54242, 28336, 74, 65544, 261557] avg=65536.0000 max=261557 max_rank=7 min=25 min_rank=2 imbalance_ratio=3.991043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[143, 131021, 47530, 131185, 37, 65596, 32, 148744] avg=65536.0000 max=148744 max_rank=7 min=32 min_rank=6 imbalance_ratio=2.269653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[143, 131021, 47530, 131185, 37, 65596, 32, 148744] avg=65536.0000 max=148744 max_rank=7 min=32 min_rank=6 imbalance_ratio=2.269653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] avg=65536.0000 max=66193 max_rank=4 min=63245 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=82551 max_load_reduction_pct=0.554987 moved_to_replicas=212374 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 65540, 64716, 65539, 63725] avg=65536.0000 max=66192 max_rank=0 min=63725 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=130162 max_load_reduction_pct=0.662895 moved_to_replicas=192278 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] avg=65536.0000 max=66193 max_rank=4 min=63245 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=82551 max_load_reduction_pct=0.554987 moved_to_replicas=212374 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[79, 114431, 25, 54242, 28336, 74, 65544, 261557] avg=65536.0000 max=261557 max_rank=7 min=25 min_rank=2 imbalance_ratio=3.991043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] avg=65536.0000 max=66192 max_rank=4 min=64481 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=195365 max_load_reduction_pct=0.746931 moved_to_replicas=243604 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] avg=65536.0000 max=66193 max_rank=4 min=63245 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=82551 max_load_reduction_pct=0.554987 moved_to_replicas=212374 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[79, 114431, 25, 54242, 28336, 74, 65544, 261557] avg=65536.0000 max=261557 max_rank=7 min=25 min_rank=2 imbalance_ratio=3.991043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] avg=65536.0000 max=66193 max_rank=4 min=63245 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=82551 max_load_reduction_pct=0.554987 moved_to_replicas=212374 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] avg=65536.0000 max=66192 max_rank=4 min=64481 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=195365 max_load_reduction_pct=0.746931 moved_to_replicas=243604 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[79, 114431, 25, 54242, 28336, 74, 65544, 261557] avg=65536.0000 max=261557 max_rank=7 min=25 min_rank=2 imbalance_ratio=3.991043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[143, 131021, 47530, 131185, 37, 65596, 32, 148744] avg=65536.0000 max=148744 max_rank=7 min=32 min_rank=6 imbalance_ratio=2.269653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] avg=65536.0000 max=66192 max_rank=4 min=64481 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=195365 max_load_reduction_pct=0.746931 moved_to_replicas=243604 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[79, 114431, 25, 54242, 28336, 74, 65544, 261557] avg=65536.0000 max=261557 max_rank=7 min=25 min_rank=2 imbalance_ratio=3.991043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65503, 75580, 67, 65490, 120855, 26, 65668, 131099] avg=65536.0000 max=131099 max_rank=7 min=26 min_rank=5 imbalance_ratio=2.000412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[79, 114431, 25, 54242, 28336, 74, 65544, 261557] avg=65536.0000 max=261557 max_rank=7 min=25 min_rank=2 imbalance_ratio=3.991043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[79, 114431, 25, 54242, 28336, 74, 65544, 261557] avg=65536.0000 max=261557 max_rank=7 min=25 min_rank=2 imbalance_ratio=3.991043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] avg=65536.0000 max=66192 max_rank=4 min=64481 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=195365 max_load_reduction_pct=0.746931 moved_to_replicas=243604 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[65136, 66192, 63245, 66192, 66193, 65596, 65542, 66192] avg=65536.0000 max=66193 max_rank=4 min=63245 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=82551 max_load_reduction_pct=0.554987 moved_to_replicas=212374 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65503, 75580, 67, 65490, 120855, 26, 65668, 131099] avg=65536.0000 max=131099 max_rank=7 min=26 min_rank=5 imbalance_ratio=2.000412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] avg=65536.0000 max=66192 max_rank=4 min=64481 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=195365 max_load_reduction_pct=0.746931 moved_to_replicas=243604 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] avg=65536.0000 max=66193 max_rank=1 min=64119 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64906 max_load_reduction_pct=0.495091 moved_to_replicas=128958 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] avg=65536.0000 max=66192 max_rank=4 min=64481 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=195365 max_load_reduction_pct=0.746931 moved_to_replicas=243604 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65503, 75580, 67, 65490, 120855, 26, 65668, 131099] avg=65536.0000 max=131099 max_rank=7 min=26 min_rank=5 imbalance_ratio=2.000412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] avg=65536.0000 max=66192 max_rank=4 min=64481 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=195365 max_load_reduction_pct=0.746931 moved_to_replicas=243604 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] avg=65536.0000 max=66193 max_rank=1 min=64119 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64906 max_load_reduction_pct=0.495091 moved_to_replicas=128958 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65503, 75580, 67, 65490, 120855, 26, 65668, 131099] avg=65536.0000 max=131099 max_rank=7 min=26 min_rank=5 imbalance_ratio=2.000412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[79, 114431, 25, 54242, 28336, 74, 65544, 261557] avg=65536.0000 max=261557 max_rank=7 min=25 min_rank=2 imbalance_ratio=3.991043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] avg=65536.0000 max=66193 max_rank=1 min=64119 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64906 max_load_reduction_pct=0.495091 moved_to_replicas=128958 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[196401, 65931, 211, 130804, 65501, 110, 65292, 38] avg=65536.0000 max=196401 max_rank=0 min=38 min_rank=7 imbalance_ratio=2.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65503, 75580, 67, 65490, 120855, 26, 65668, 131099] avg=65536.0000 max=131099 max_rank=7 min=26 min_rank=5 imbalance_ratio=2.000412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65503, 75580, 67, 65490, 120855, 26, 65668, 131099] avg=65536.0000 max=131099 max_rank=7 min=26 min_rank=5 imbalance_ratio=2.000412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65503, 75580, 67, 65490, 120855, 26, 65668, 131099] avg=65536.0000 max=131099 max_rank=7 min=26 min_rank=5 imbalance_ratio=2.000412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] avg=65536.0000 max=66193 max_rank=1 min=64119 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64906 max_load_reduction_pct=0.495091 moved_to_replicas=128958 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[64481, 66191, 65518, 64626, 66192, 65544, 65544, 66192] avg=65536.0000 max=66192 max_rank=4 min=64481 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=195365 max_load_reduction_pct=0.746931 moved_to_replicas=243604 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[196401, 65931, 211, 130804, 65501, 110, 65292, 38] avg=65536.0000 max=196401 max_rank=0 min=38 min_rank=7 imbalance_ratio=2.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] avg=65536.0000 max=66192 max_rank=0 min=64819 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130209 max_load_reduction_pct=0.662975 moved_to_replicas=194821 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] avg=65536.0000 max=66193 max_rank=1 min=64119 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64906 max_load_reduction_pct=0.495091 moved_to_replicas=128958 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] avg=65536.0000 max=66193 max_rank=1 min=64119 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64906 max_load_reduction_pct=0.495091 moved_to_replicas=128958 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[196401, 65931, 211, 130804, 65501, 110, 65292, 38] avg=65536.0000 max=196401 max_rank=0 min=38 min_rank=7 imbalance_ratio=2.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] avg=65536.0000 max=66193 max_rank=1 min=64119 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64906 max_load_reduction_pct=0.495091 moved_to_replicas=128958 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] avg=65536.0000 max=66192 max_rank=0 min=64819 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130209 max_load_reduction_pct=0.662975 moved_to_replicas=194821 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[196401, 65931, 211, 130804, 65501, 110, 65292, 38] avg=65536.0000 max=196401 max_rank=0 min=38 min_rank=7 imbalance_ratio=2.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65503, 75580, 67, 65490, 120855, 26, 65668, 131099] avg=65536.0000 max=131099 max_rank=7 min=26 min_rank=5 imbalance_ratio=2.000412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[124158, 65512, 246405, 108, 913, 65508, 39, 21645] avg=65536.0000 max=246405 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.759842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] avg=65536.0000 max=66192 max_rank=0 min=64819 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130209 max_load_reduction_pct=0.662975 moved_to_replicas=194821 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[196401, 65931, 211, 130804, 65501, 110, 65292, 38] avg=65536.0000 max=196401 max_rank=0 min=38 min_rank=7 imbalance_ratio=2.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[196401, 65931, 211, 130804, 65501, 110, 65292, 38] avg=65536.0000 max=196401 max_rank=0 min=38 min_rank=7 imbalance_ratio=2.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[196401, 65931, 211, 130804, 65501, 110, 65292, 38] avg=65536.0000 max=196401 max_rank=0 min=38 min_rank=7 imbalance_ratio=2.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] avg=65536.0000 max=66192 max_rank=0 min=64819 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130209 max_load_reduction_pct=0.662975 moved_to_replicas=194821 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65503, 66193, 64119, 65490, 66190, 64933, 65668, 66192] avg=65536.0000 max=66193 max_rank=1 min=64119 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64906 max_load_reduction_pct=0.495091 moved_to_replicas=128958 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180213 max_load_reduction_pct=0.731369 moved_to_replicas=238179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[124158, 65512, 246405, 108, 913, 65508, 39, 21645] avg=65536.0000 max=246405 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.759842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] avg=65536.0000 max=66192 max_rank=0 min=64819 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130209 max_load_reduction_pct=0.662975 moved_to_replicas=194821 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] avg=65536.0000 max=66192 max_rank=0 min=64819 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130209 max_load_reduction_pct=0.662975 moved_to_replicas=194821 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[124158, 65512, 246405, 108, 913, 65508, 39, 21645] avg=65536.0000 max=246405 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.759842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] avg=65536.0000 max=66192 max_rank=0 min=64819 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130209 max_load_reduction_pct=0.662975 moved_to_replicas=194821 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180213 max_load_reduction_pct=0.731369 moved_to_replicas=238179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[124158, 65512, 246405, 108, 913, 65508, 39, 21645] avg=65536.0000 max=246405 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.759842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[25, 131543, 49, 186, 62567, 184579, 14662, 130677] avg=65536.0000 max=184579 max_rank=5 min=25 min_rank=0 imbalance_ratio=2.816452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[196401, 65931, 211, 130804, 65501, 110, 65292, 38] avg=65536.0000 max=196401 max_rank=0 min=38 min_rank=7 imbalance_ratio=2.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180213 max_load_reduction_pct=0.731369 moved_to_replicas=238179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[124158, 65512, 246405, 108, 913, 65508, 39, 21645] avg=65536.0000 max=246405 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.759842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[124158, 65512, 246405, 108, 913, 65508, 39, 21645] avg=65536.0000 max=246405 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.759842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180213 max_load_reduction_pct=0.731369 moved_to_replicas=238179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[124158, 65512, 246405, 108, 913, 65508, 39, 21645] avg=65536.0000 max=246405 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.759842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62567 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=118386 max_load_reduction_pct=0.641384 moved_to_replicas=248223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[25, 131543, 49, 186, 62567, 184579, 14662, 130677] avg=65536.0000 max=184579 max_rank=5 min=25 min_rank=0 imbalance_ratio=2.816452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[66192, 65931, 64823, 66192, 65501, 64819, 65292, 65538] avg=65536.0000 max=66192 max_rank=0 min=64819 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130209 max_load_reduction_pct=0.662975 moved_to_replicas=194821 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180213 max_load_reduction_pct=0.731369 moved_to_replicas=238179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[25, 131543, 49, 186, 62567, 184579, 14662, 130677] avg=65536.0000 max=184579 max_rank=5 min=25 min_rank=0 imbalance_ratio=2.816452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180213 max_load_reduction_pct=0.731369 moved_to_replicas=238179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180213 max_load_reduction_pct=0.731369 moved_to_replicas=238179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62567 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=118386 max_load_reduction_pct=0.641384 moved_to_replicas=248223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[25, 131543, 49, 186, 62567, 184579, 14662, 130677] avg=65536.0000 max=184579 max_rank=5 min=25 min_rank=0 imbalance_ratio=2.816452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[130909, 65641, 131055, 139, 65566, 67, 65357, 65554] avg=65536.0000 max=131055 max_rank=2 min=67 min_rank=5 imbalance_ratio=1.999741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62567 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=118386 max_load_reduction_pct=0.641384 moved_to_replicas=248223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[124158, 65512, 246405, 108, 913, 65508, 39, 21645] avg=65536.0000 max=246405 max_rank=2 min=39 min_rank=6 imbalance_ratio=3.759842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[25, 131543, 49, 186, 62567, 184579, 14662, 130677] avg=65536.0000 max=184579 max_rank=5 min=25 min_rank=0 imbalance_ratio=2.816452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62567 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=118386 max_load_reduction_pct=0.641384 moved_to_replicas=248223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[25, 131543, 49, 186, 62567, 184579, 14662, 130677] avg=65536.0000 max=184579 max_rank=5 min=25 min_rank=0 imbalance_ratio=2.816452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[130909, 65641, 131055, 139, 65566, 67, 65357, 65554] avg=65536.0000 max=131055 max_rank=2 min=67 min_rank=5 imbalance_ratio=1.999741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[25, 131543, 49, 186, 62567, 184579, 14662, 130677] avg=65536.0000 max=184579 max_rank=5 min=25 min_rank=0 imbalance_ratio=2.816452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] avg=65536.0000 max=66192 max_rank=0 min=64856 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64863 max_load_reduction_pct=0.494930 moved_to_replicas=129580 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[66192, 65512, 66192, 65569, 63591, 65508, 65532, 66192] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180213 max_load_reduction_pct=0.731369 moved_to_replicas=238179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62567 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=118386 max_load_reduction_pct=0.641384 moved_to_replicas=248223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[130909, 65641, 131055, 139, 65566, 67, 65357, 65554] avg=65536.0000 max=131055 max_rank=2 min=67 min_rank=5 imbalance_ratio=1.999741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62567 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=118386 max_load_reduction_pct=0.641384 moved_to_replicas=248223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] avg=65536.0000 max=66192 max_rank=0 min=64856 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64863 max_load_reduction_pct=0.494930 moved_to_replicas=129580 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62567 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=118386 max_load_reduction_pct=0.641384 moved_to_replicas=248223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[130909, 65641, 131055, 139, 65566, 67, 65357, 65554] avg=65536.0000 max=131055 max_rank=2 min=67 min_rank=5 imbalance_ratio=1.999741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65561, 104, 131326, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=131326 max_rank=2 min=104 min_rank=1 imbalance_ratio=2.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] avg=65536.0000 max=66192 max_rank=0 min=64856 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64863 max_load_reduction_pct=0.494930 moved_to_replicas=129580 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[25, 131543, 49, 186, 62567, 184579, 14662, 130677] avg=65536.0000 max=184579 max_rank=5 min=25 min_rank=0 imbalance_ratio=2.816452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[130909, 65641, 131055, 139, 65566, 67, 65357, 65554] avg=65536.0000 max=131055 max_rank=2 min=67 min_rank=5 imbalance_ratio=1.999741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] avg=65536.0000 max=66192 max_rank=0 min=64856 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64863 max_load_reduction_pct=0.494930 moved_to_replicas=129580 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=66192 max_rank=2 min=65140 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65134 max_load_reduction_pct=0.495972 moved_to_replicas=65134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65561, 104, 131326, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=131326 max_rank=2 min=104 min_rank=1 imbalance_ratio=2.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[130909, 65641, 131055, 139, 65566, 67, 65357, 65554] avg=65536.0000 max=131055 max_rank=2 min=67 min_rank=5 imbalance_ratio=1.999741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[130909, 65641, 131055, 139, 65566, 67, 65357, 65554] avg=65536.0000 max=131055 max_rank=2 min=67 min_rank=5 imbalance_ratio=1.999741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65522, 66192, 65893, 65537, 62567, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62567 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=118386 max_load_reduction_pct=0.641384 moved_to_replicas=248223 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] avg=65536.0000 max=66192 max_rank=0 min=64856 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64863 max_load_reduction_pct=0.494930 moved_to_replicas=129580 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65561, 104, 131326, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=131326 max_rank=2 min=104 min_rank=1 imbalance_ratio=2.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=66192 max_rank=2 min=65140 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65134 max_load_reduction_pct=0.495972 moved_to_replicas=65134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] avg=65536.0000 max=66192 max_rank=0 min=64856 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64863 max_load_reduction_pct=0.494930 moved_to_replicas=129580 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] avg=65536.0000 max=66192 max_rank=0 min=64856 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64863 max_load_reduction_pct=0.494930 moved_to_replicas=129580 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[23, 131357, 65490, 195553, 374, 114, 131132, 245] avg=65536.0000 max=195553 max_rank=3 min=23 min_rank=0 imbalance_ratio=2.983902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65561, 104, 131326, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=131326 max_rank=2 min=104 min_rank=1 imbalance_ratio=2.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=66192 max_rank=2 min=65140 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65134 max_load_reduction_pct=0.495972 moved_to_replicas=65134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[130909, 65641, 131055, 139, 65566, 67, 65357, 65554] avg=65536.0000 max=131055 max_rank=2 min=67 min_rank=5 imbalance_ratio=1.999741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65561, 104, 131326, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=131326 max_rank=2 min=104 min_rank=1 imbalance_ratio=2.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] avg=65536.0000 max=66192 max_rank=1 min=63983 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129361 max_load_reduction_pct=0.661514 moved_to_replicas=259466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[23, 131357, 65490, 195553, 374, 114, 131132, 245] avg=65536.0000 max=195553 max_rank=3 min=23 min_rank=0 imbalance_ratio=2.983902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=66192 max_rank=2 min=65140 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65134 max_load_reduction_pct=0.495972 moved_to_replicas=65134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65561, 104, 131326, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=131326 max_rank=2 min=104 min_rank=1 imbalance_ratio=2.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65561, 104, 131326, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=131326 max_rank=2 min=104 min_rank=1 imbalance_ratio=2.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[66192, 65641, 66192, 64856, 65566, 64930, 65357, 65554] avg=65536.0000 max=66192 max_rank=0 min=64856 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64863 max_load_reduction_pct=0.494930 moved_to_replicas=129580 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=66192 max_rank=2 min=65140 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65134 max_load_reduction_pct=0.495972 moved_to_replicas=65134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[23, 131357, 65490, 195553, 374, 114, 131132, 245] avg=65536.0000 max=195553 max_rank=3 min=23 min_rank=0 imbalance_ratio=2.983902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] avg=65536.0000 max=66192 max_rank=1 min=63983 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129361 max_load_reduction_pct=0.661514 moved_to_replicas=259466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=66192 max_rank=2 min=65140 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65134 max_load_reduction_pct=0.495972 moved_to_replicas=65134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65517, 1330, 260669, 60034, 65554, 71136, 33, 15] avg=65536.0000 max=260669 max_rank=2 min=15 min_rank=7 imbalance_ratio=3.977493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=66192 max_rank=2 min=65140 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65134 max_load_reduction_pct=0.495972 moved_to_replicas=65134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[23, 131357, 65490, 195553, 374, 114, 131132, 245] avg=65536.0000 max=195553 max_rank=3 min=23 min_rank=0 imbalance_ratio=2.983902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] avg=65536.0000 max=66192 max_rank=1 min=63983 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129361 max_load_reduction_pct=0.661514 moved_to_replicas=259466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65561, 104, 131326, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=131326 max_rank=2 min=104 min_rank=1 imbalance_ratio=2.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[23, 131357, 65490, 195553, 374, 114, 131132, 245] avg=65536.0000 max=195553 max_rank=3 min=23 min_rank=0 imbalance_ratio=2.983902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] avg=65536.0000 max=66192 max_rank=2 min=64962 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194477 max_load_reduction_pct=0.746069 moved_to_replicas=199421 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] avg=65536.0000 max=66192 max_rank=1 min=63983 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129361 max_load_reduction_pct=0.661514 moved_to_replicas=259466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65517, 1330, 260669, 60034, 65554, 71136, 33, 15] avg=65536.0000 max=260669 max_rank=2 min=15 min_rank=7 imbalance_ratio=3.977493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[23, 131357, 65490, 195553, 374, 114, 131132, 245] avg=65536.0000 max=195553 max_rank=3 min=23 min_rank=0 imbalance_ratio=2.983902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[23, 131357, 65490, 195553, 374, 114, 131132, 245] avg=65536.0000 max=195553 max_rank=3 min=23 min_rank=0 imbalance_ratio=2.983902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65561, 65238, 66192, 65509, 65579, 65531, 65538, 65140] avg=65536.0000 max=66192 max_rank=2 min=65140 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65134 max_load_reduction_pct=0.495972 moved_to_replicas=65134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] avg=65536.0000 max=66192 max_rank=1 min=63983 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129361 max_load_reduction_pct=0.661514 moved_to_replicas=259466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65517, 1330, 260669, 60034, 65554, 71136, 33, 15] avg=65536.0000 max=260669 max_rank=2 min=15 min_rank=7 imbalance_ratio=3.977493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] avg=65536.0000 max=66192 max_rank=2 min=64962 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194477 max_load_reduction_pct=0.746069 moved_to_replicas=199421 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] avg=65536.0000 max=66192 max_rank=1 min=63983 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129361 max_load_reduction_pct=0.661514 moved_to_replicas=259466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] avg=65536.0000 max=66192 max_rank=1 min=63983 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129361 max_load_reduction_pct=0.661514 moved_to_replicas=259466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65517, 1330, 260669, 60034, 65554, 71136, 33, 15] avg=65536.0000 max=260669 max_rank=2 min=15 min_rank=7 imbalance_ratio=3.977493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] avg=65536.0000 max=66192 max_rank=2 min=64962 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194477 max_load_reduction_pct=0.746069 moved_to_replicas=199421 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[23, 131357, 65490, 195553, 374, 114, 131132, 245] avg=65536.0000 max=195553 max_rank=3 min=23 min_rank=0 imbalance_ratio=2.983902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65517, 1330, 260669, 60034, 65554, 71136, 33, 15] avg=65536.0000 max=260669 max_rank=2 min=15 min_rank=7 imbalance_ratio=3.977493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] avg=65536.0000 max=66192 max_rank=2 min=64962 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194477 max_load_reduction_pct=0.746069 moved_to_replicas=199421 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65517, 1330, 260669, 60034, 65554, 71136, 33, 15] avg=65536.0000 max=260669 max_rank=2 min=15 min_rank=7 imbalance_ratio=3.977493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65517, 1330, 260669, 60034, 65554, 71136, 33, 15] avg=65536.0000 max=260669 max_rank=2 min=15 min_rank=7 imbalance_ratio=3.977493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[65515, 66192, 65490, 66192, 65314, 63983, 66192, 65410] avg=65536.0000 max=66192 max_rank=1 min=63983 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=129361 max_load_reduction_pct=0.661514 moved_to_replicas=259466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] avg=65536.0000 max=66192 max_rank=2 min=64962 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194477 max_load_reduction_pct=0.746069 moved_to_replicas=199421 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] avg=65536.0000 max=66192 max_rank=2 min=64962 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194477 max_load_reduction_pct=0.746069 moved_to_replicas=199421 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] avg=65536.0000 max=66192 max_rank=2 min=64962 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194477 max_load_reduction_pct=0.746069 moved_to_replicas=199421 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65517, 1330, 260669, 60034, 65554, 71136, 33, 15] avg=65536.0000 max=260669 max_rank=2 min=15 min_rank=7 imbalance_ratio=3.977493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65517, 64962, 66192, 64978, 65554, 66192, 65397, 65496] avg=65536.0000 max=66192 max_rank=2 min=64962 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194477 max_load_reduction_pct=0.746069 moved_to_replicas=199421 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=12 total_expected_recv=783609 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190720 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=12 total_expected_recv=786240 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188160 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=12 total_expected_recv=790628 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184064 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=12 total_expected_recv=783264 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190720 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=12 total_expected_recv=787525 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=12 total_expected_recv=788015 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=12 total_expected_recv=788196 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186752 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=12 total_expected_recv=783979 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190592 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 15:44:41] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 341.8 | throughput per GPU (TFLOP/s/GPU): 529.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.034307E+01 | loss scale: 1.0 | grad norm: 45.887 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=81920 expected_recv=63633 expected_overflow=0 exact_recv_per_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=81920 expected_recv=64250 expected_overflow=0 exact_recv_per_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=81920 expected_recv=63633 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=81920 expected_recv=64250 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65451 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65446 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=81920 expected_recv=62963 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=81920 expected_recv=64263 expected_overflow=0 exact_recv_per_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=81920 expected_recv=62963 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=81920 expected_recv=64263 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65431 expected_overflow=0 exact_recv_per_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=81920 expected_recv=64898 expected_overflow=0 exact_recv_per_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65431 actual_valid_recv=66304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65435 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=81920 expected_recv=64898 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65577 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65579 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66114 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64325 expected_overflow=0 exact_recv_per_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64872 actual_valid_recv=65280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64425 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64878 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66114 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64870 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64325 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64425 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64878 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64878 expected_overflow=0 exact_recv_per_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65445 expected_overflow=0 exact_recv_per_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64878 actual_valid_recv=65664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65445 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=81920 expected_recv=64870 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65425 expected_overflow=0 exact_recv_per_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65425 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=81920 expected_recv=64208 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65551 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=81920 expected_recv=64237 expected_overflow=0 exact_recv_per_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65495 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=81920 expected_recv=64237 actual_valid_recv=64768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=81920 expected_recv=64208 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=81920 expected_recv=63251 expected_overflow=0 exact_recv_per_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65561 expected_overflow=0 exact_recv_per_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64578 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64955 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64578 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64491 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64955 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=81920 expected_recv=64491 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65504 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=81920 expected_recv=63251 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65561 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65204 expected_overflow=0 exact_recv_per_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65204 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65496 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65701 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65590 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65550 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65701 actual_valid_recv=66304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65590 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65551 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=81920 expected_recv=64077 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65599 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65624 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=81920 expected_recv=64077 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65556 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=81920 expected_recv=63934 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65599 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=81920 expected_recv=63956 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=81920 expected_recv=63934 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=81920 expected_recv=64441 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=81920 expected_recv=63956 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=81920 expected_recv=64441 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65210 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=81920 expected_recv=64818 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=81920 expected_recv=64023 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65210 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=81920 expected_recv=64818 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=81920 expected_recv=64023 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=81920 expected_recv=64885 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65519 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65011 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=81920 expected_recv=64885 actual_valid_recv=65280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=81920 expected_recv=62426 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65391 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=81920 expected_recv=62426 actual_valid_recv=62976 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65391 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65011 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65188 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65188 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=81920 expected_recv=64436 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=81920 expected_recv=64436 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64906 expected_overflow=0 exact_recv_per_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65516 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64916 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64916 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66128 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65598 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66128 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=81920 expected_recv=63555 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64906 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64738 expected_overflow=0 exact_recv_per_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=81920 expected_recv=63555 actual_valid_recv=63872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=81920 expected_recv=64959 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65363 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64738 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65618 expected_overflow=0 exact_recv_per_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65618 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64934 expected_overflow=0 exact_recv_per_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=81920 expected_recv=64937 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=81920 expected_recv=64934 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=81920 expected_recv=64937 actual_valid_recv=65152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=81920 expected_recv=64959 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65568 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=65792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65363 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65405 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65548 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65405 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65610 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65610 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65559 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65593 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65593 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65578 expected_overflow=0 exact_recv_per_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65578 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64882 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65123 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66054 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64882 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65123 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66054 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65572 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64081 expected_overflow=0 exact_recv_per_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64081 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=81920 expected_recv=63929 expected_overflow=0 exact_recv_per_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=81920 expected_recv=63929 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64501 expected_overflow=0 exact_recv_per_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65512 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64923 expected_overflow=0 exact_recv_per_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=81920 expected_recv=64501 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65550 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64923 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66100 expected_overflow=0 exact_recv_per_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=81920 expected_recv=66100 actual_valid_recv=66560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65570 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65545 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64761 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[65446, 133693, 63633, 20624, 161786, 2942, 30461, 45703] avg=65536.0000 max=161786 max_rank=4 min=2942 min_rank=5 imbalance_ratio=2.468658 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65512 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64761 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65550 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64966 expected_overflow=0 exact_recv_per_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64903 expected_overflow=0 exact_recv_per_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64966 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[65446, 133693, 63633, 20624, 161786, 2942, 30461, 45703] avg=65536.0000 max=161786 max_rank=4 min=2942 min_rank=5 imbalance_ratio=2.468658 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=81920 expected_recv=64903 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[65446, 133693, 63633, 20624, 161786, 2942, 30461, 45703] avg=65536.0000 max=161786 max_rank=4 min=2942 min_rank=5 imbalance_ratio=2.468658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63633 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=95594 max_load_reduction_pct=0.590867 moved_to_replicas=163095 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[65451, 134149, 65100, 10191, 161053, 3032, 30689, 54623] avg=65536.0000 max=161053 max_rank=4 min=3032 min_rank=5 imbalance_ratio=2.457474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[65451, 66192, 65100, 66192, 66193, 66192, 66191, 62777] avg=65536.0000 max=66193 max_rank=4 min=62777 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=94860 max_load_reduction_pct=0.588999 moved_to_replicas=162818 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65579, 196582, 23883, 131052, 65435, 18, 39221, 2518] avg=65536.0000 max=196582 max_rank=1 min=18 min_rank=5 imbalance_ratio=2.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65579, 66192, 66191, 66192, 65435, 65544, 62963, 66192] avg=65536.0000 max=66192 max_rank=1 min=62963 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130390 max_load_reduction_pct=0.663286 moved_to_replicas=195250 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 131049, 131053, 21, 9, 31, 130586] avg=65536.0000 max=131053 max_rank=3 min=9 min_rank=5 imbalance_ratio=1.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65425, 66114, 66192, 66192, 64878, 64870, 64425, 66192] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64861 max_load_reduction_pct=0.494922 moved_to_replicas=194112 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[19, 65535, 21, 65551, 128354, 10, 86, 264712] avg=65536.0000 max=264712 max_rank=7 min=10 min_rank=5 imbalance_ratio=4.039185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65535, 65537, 65551, 66192, 65534, 64208, 66192] avg=65536.0000 max=66192 max_rank=4 min=64208 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=198520 max_load_reduction_pct=0.749947 moved_to_replicas=260682 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[200512, 64955, 56477, 130890, 71123, 16, 168, 147] avg=65536.0000 max=200512 max_rank=0 min=16 min_rank=5 imbalance_ratio=3.059570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[66192, 64955, 64578, 66192, 66192, 65496, 66192, 64491] avg=65536.0000 max=66192 max_rank=0 min=64491 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134320 max_load_reduction_pct=0.669885 moved_to_replicas=203949 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[76203, 18, 55579, 32, 261229, 65549, 65599, 79] avg=65536.0000 max=261229 max_rank=4 min=18 min_rank=1 imbalance_ratio=3.986038 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65538, 65590, 65551, 66192, 65549, 65599, 64077] avg=65536.0000 max=66192 max_rank=0 min=64077 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=195037 max_load_reduction_pct=0.746613 moved_to_replicas=205048 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[130216, 196540, 63956, 6680, 18, 1455, 60213, 65210] avg=65536.0000 max=196540 max_rank=1 min=18 min_rank=4 imbalance_ratio=2.998962 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 63956, 66192, 65536, 66192, 64818, 65210] avg=65536.0000 max=66192 max_rank=0 min=63956 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=130348 max_load_reduction_pct=0.663214 moved_to_replicas=194372 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[156261, 100514, 24, 529, 5616, 130282, 8, 131054] avg=65536.0000 max=156261 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.384354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[66191, 66191, 62426, 65391, 66192, 66192, 65513, 66192] avg=65536.0000 max=66192 max_rank=4 min=62426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=90069 max_load_reduction_pct=0.576401 moved_to_replicas=253343 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[196473, 6195, 65515, 130, 103, 193301, 61943, 628] avg=65536.0000 max=196473 max_rank=0 min=103 min_rank=4 imbalance_ratio=2.997940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[66192, 66192, 65515, 64916, 65598, 66192, 63555, 66128] avg=65536.0000 max=66192 max_rank=0 min=63555 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130281 max_load_reduction_pct=0.663099 moved_to_replicas=257390 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131006, 65558, 145, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=131006 max_rank=0 min=145 min_rank=2 imbalance_ratio=1.998993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65558, 64959, 65363, 65526, 65521, 65559, 65610] avg=65536.0000 max=66192 max_rank=0 min=64959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64814 max_load_reduction_pct=0.494741 moved_to_replicas=64814 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[64081, 131037, 130438, 131315, 37, 1808, 65572, 0] avg=65536.0000 max=131315 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.003708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[64081, 66192, 66192, 66192, 64882, 66054, 65572, 65123] avg=65536.0000 max=66192 max_rank=1 min=64081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.495930 moved_to_replicas=194214 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[424, 65570, 196227, 66267, 52, 64686, 65550, 65512] avg=65536.0000 max=196227 max_rank=2 min=52 min_rank=4 imbalance_ratio=2.994186 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[64966, 65570, 66192, 66192, 65545, 64761, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64761 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130035 max_load_reduction_pct=0.662676 moved_to_replicas=130110 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=12 total_expected_recv=790421 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=12 total_expected_recv=790116 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=12 total_expected_recv=788361 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187520 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=12 total_expected_recv=782428 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194304 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=12 total_expected_recv=788097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187392 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=12 total_expected_recv=783696 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192512 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=12 total_expected_recv=780145 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=196096 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=12 total_expected_recv=788192 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186624 peak_demand=66192 required_capacity=1.010010 - [2026-08-26 14:59:20] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 341.3 | throughput per GPU (TFLOP/s/GPU): 529.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.026134E+01 | loss scale: 1.0 | grad norm: 6.258 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[65446, 133693, 63633, 20624, 161786, 2942, 30461, 45703] avg=65536.0000 max=161786 max_rank=4 min=2942 min_rank=5 imbalance_ratio=2.468658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[65446, 133693, 63633, 20624, 161786, 2942, 30461, 45703] avg=65536.0000 max=161786 max_rank=4 min=2942 min_rank=5 imbalance_ratio=2.468658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63633 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=95594 max_load_reduction_pct=0.590867 moved_to_replicas=163095 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63633 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=95594 max_load_reduction_pct=0.590867 moved_to_replicas=163095 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[65446, 133693, 63633, 20624, 161786, 2942, 30461, 45703] avg=65536.0000 max=161786 max_rank=4 min=2942 min_rank=5 imbalance_ratio=2.468658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[65446, 133693, 63633, 20624, 161786, 2942, 30461, 45703] avg=65536.0000 max=161786 max_rank=4 min=2942 min_rank=5 imbalance_ratio=2.468658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63633 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=95594 max_load_reduction_pct=0.590867 moved_to_replicas=163095 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63633 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=95594 max_load_reduction_pct=0.590867 moved_to_replicas=163095 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65577, 196583, 11364, 131051, 65431, 17, 54231, 34] avg=65536.0000 max=196583 max_rank=1 min=17 min_rank=5 imbalance_ratio=2.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[65446, 133693, 63633, 20624, 161786, 2942, 30461, 45703] avg=65536.0000 max=161786 max_rank=4 min=2942 min_rank=5 imbalance_ratio=2.468658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65577, 196583, 11364, 131051, 65431, 17, 54231, 34] avg=65536.0000 max=196583 max_rank=1 min=17 min_rank=5 imbalance_ratio=2.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63633 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=95594 max_load_reduction_pct=0.590867 moved_to_replicas=163095 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63633 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=95594 max_load_reduction_pct=0.590867 moved_to_replicas=163095 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] avg=65536.0000 max=66192 max_rank=1 min=64263 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130391 max_load_reduction_pct=0.663287 moved_to_replicas=195250 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65577, 196583, 11364, 131051, 65431, 17, 54231, 34] avg=65536.0000 max=196583 max_rank=1 min=17 min_rank=5 imbalance_ratio=2.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[65446, 66192, 63633, 64250, 66191, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63633 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=95594 max_load_reduction_pct=0.590867 moved_to_replicas=163095 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65577, 196583, 11364, 131051, 65431, 17, 54231, 34] avg=65536.0000 max=196583 max_rank=1 min=17 min_rank=5 imbalance_ratio=2.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65577, 196583, 11364, 131051, 65431, 17, 54231, 34] avg=65536.0000 max=196583 max_rank=1 min=17 min_rank=5 imbalance_ratio=2.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] avg=65536.0000 max=66192 max_rank=1 min=64263 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130391 max_load_reduction_pct=0.663287 moved_to_replicas=195250 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] avg=65536.0000 max=66192 max_rank=1 min=64263 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130391 max_load_reduction_pct=0.663287 moved_to_replicas=195250 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65577, 196583, 11364, 131051, 65431, 17, 54231, 34] avg=65536.0000 max=196583 max_rank=1 min=17 min_rank=5 imbalance_ratio=2.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65577, 196583, 11364, 131051, 65431, 17, 54231, 34] avg=65536.0000 max=196583 max_rank=1 min=17 min_rank=5 imbalance_ratio=2.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] avg=65536.0000 max=66192 max_rank=1 min=64263 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130391 max_load_reduction_pct=0.663287 moved_to_replicas=195250 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66299, 131052, 131051, 19, 12, 31, 130379] avg=65536.0000 max=131052 max_rank=2 min=12 min_rank=5 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] avg=65536.0000 max=66192 max_rank=1 min=64263 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130391 max_load_reduction_pct=0.663287 moved_to_replicas=195250 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65577, 196583, 11364, 131051, 65431, 17, 54231, 34] avg=65536.0000 max=196583 max_rank=1 min=17 min_rank=5 imbalance_ratio=2.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66299, 131052, 131051, 19, 12, 31, 130379] avg=65536.0000 max=131052 max_rank=2 min=12 min_rank=5 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] avg=65536.0000 max=66192 max_rank=1 min=64263 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130391 max_load_reduction_pct=0.663287 moved_to_replicas=195250 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] avg=65536.0000 max=66192 max_rank=1 min=64263 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130391 max_load_reduction_pct=0.663287 moved_to_replicas=195250 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] avg=65536.0000 max=66192 max_rank=1 min=64325 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=194013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66299, 131052, 131051, 19, 12, 31, 130379] avg=65536.0000 max=131052 max_rank=2 min=12 min_rank=5 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65577, 66192, 66191, 66192, 65431, 65544, 64263, 64898] avg=65536.0000 max=66192 max_rank=1 min=64263 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130391 max_load_reduction_pct=0.663287 moved_to_replicas=195250 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66299, 131052, 131051, 19, 12, 31, 130379] avg=65536.0000 max=131052 max_rank=2 min=12 min_rank=5 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] avg=65536.0000 max=66192 max_rank=1 min=64325 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=194013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66299, 131052, 131051, 19, 12, 31, 130379] avg=65536.0000 max=131052 max_rank=2 min=12 min_rank=5 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] avg=65536.0000 max=66192 max_rank=1 min=64325 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=194013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66299, 131052, 131051, 19, 12, 31, 130379] avg=65536.0000 max=131052 max_rank=2 min=12 min_rank=5 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66299, 131052, 131051, 19, 12, 31, 130379] avg=65536.0000 max=131052 max_rank=2 min=12 min_rank=5 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[19, 65545, 21, 65553, 130341, 11, 88, 262710] avg=65536.0000 max=262710 max_rank=7 min=11 min_rank=5 imbalance_ratio=4.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] avg=65536.0000 max=66192 max_rank=1 min=64325 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=194013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] avg=65536.0000 max=66192 max_rank=1 min=64325 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=194013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66299, 131052, 131051, 19, 12, 31, 130379] avg=65536.0000 max=131052 max_rank=2 min=12 min_rank=5 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[19, 65545, 21, 65553, 130341, 11, 88, 262710] avg=65536.0000 max=262710 max_rank=7 min=11 min_rank=5 imbalance_ratio=4.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] avg=65536.0000 max=66192 max_rank=1 min=64325 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=194013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] avg=65536.0000 max=66192 max_rank=4 min=64237 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=196518 max_load_reduction_pct=0.748042 moved_to_replicas=260667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] avg=65536.0000 max=66192 max_rank=1 min=64325 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=194013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[19, 65545, 21, 65553, 130341, 11, 88, 262710] avg=65536.0000 max=262710 max_rank=7 min=11 min_rank=5 imbalance_ratio=4.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65445, 66192, 66192, 66192, 64878, 64872, 64325, 66192] avg=65536.0000 max=66192 max_rank=1 min=64325 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=194013 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] avg=65536.0000 max=66192 max_rank=4 min=64237 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=196518 max_load_reduction_pct=0.748042 moved_to_replicas=260667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[19, 65545, 21, 65553, 130341, 11, 88, 262710] avg=65536.0000 max=262710 max_rank=7 min=11 min_rank=5 imbalance_ratio=4.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[19, 65545, 21, 65553, 130341, 11, 88, 262710] avg=65536.0000 max=262710 max_rank=7 min=11 min_rank=5 imbalance_ratio=4.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] avg=65536.0000 max=66192 max_rank=4 min=64237 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=196518 max_load_reduction_pct=0.748042 moved_to_replicas=260667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[19, 65545, 21, 65553, 130341, 11, 88, 262710] avg=65536.0000 max=262710 max_rank=7 min=11 min_rank=5 imbalance_ratio=4.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[206242, 65204, 54631, 130948, 66799, 14, 161, 289] avg=65536.0000 max=206242 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.147003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[19, 65545, 21, 65553, 130341, 11, 88, 262710] avg=65536.0000 max=262710 max_rank=7 min=11 min_rank=5 imbalance_ratio=4.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] avg=65536.0000 max=66192 max_rank=4 min=64237 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=196518 max_load_reduction_pct=0.748042 moved_to_replicas=260667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] avg=65536.0000 max=66192 max_rank=4 min=64237 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=196518 max_load_reduction_pct=0.748042 moved_to_replicas=260667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[19, 65545, 21, 65553, 130341, 11, 88, 262710] avg=65536.0000 max=262710 max_rank=7 min=11 min_rank=5 imbalance_ratio=4.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[206242, 65204, 54631, 130948, 66799, 14, 161, 289] avg=65536.0000 max=206242 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.147003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] avg=65536.0000 max=66192 max_rank=0 min=63251 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=140050 max_load_reduction_pct=0.679057 moved_to_replicas=205413 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[206242, 65204, 54631, 130948, 66799, 14, 161, 289] avg=65536.0000 max=206242 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.147003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] avg=65536.0000 max=66192 max_rank=4 min=64237 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=196518 max_load_reduction_pct=0.748042 moved_to_replicas=260667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] avg=65536.0000 max=66192 max_rank=4 min=64237 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=196518 max_load_reduction_pct=0.748042 moved_to_replicas=260667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] avg=65536.0000 max=66192 max_rank=0 min=63251 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=140050 max_load_reduction_pct=0.679057 moved_to_replicas=205413 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65539, 65545, 65495, 65553, 66192, 65535, 64237, 66192] avg=65536.0000 max=66192 max_rank=4 min=64237 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=196518 max_load_reduction_pct=0.748042 moved_to_replicas=260667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[206242, 65204, 54631, 130948, 66799, 14, 161, 289] avg=65536.0000 max=206242 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.147003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[206242, 65204, 54631, 130948, 66799, 14, 161, 289] avg=65536.0000 max=206242 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.147003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] avg=65536.0000 max=66192 max_rank=0 min=63251 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=140050 max_load_reduction_pct=0.679057 moved_to_replicas=205413 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[126310, 19, 5583, 37, 261107, 65550, 65624, 58] avg=65536.0000 max=261107 max_rank=4 min=19 min_rank=1 imbalance_ratio=3.984177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[206242, 65204, 54631, 130948, 66799, 14, 161, 289] avg=65536.0000 max=206242 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.147003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[206242, 65204, 54631, 130948, 66799, 14, 161, 289] avg=65536.0000 max=206242 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.147003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] avg=65536.0000 max=66192 max_rank=0 min=63251 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=140050 max_load_reduction_pct=0.679057 moved_to_replicas=205413 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] avg=65536.0000 max=66192 max_rank=0 min=63251 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=140050 max_load_reduction_pct=0.679057 moved_to_replicas=205413 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[126310, 19, 5583, 37, 261107, 65550, 65624, 58] avg=65536.0000 max=261107 max_rank=4 min=19 min_rank=1 imbalance_ratio=3.984177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[206242, 65204, 54631, 130948, 66799, 14, 161, 289] avg=65536.0000 max=206242 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.147003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] avg=65536.0000 max=66192 max_rank=0 min=63934 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=194915 max_load_reduction_pct=0.746495 moved_to_replicas=255033 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[126310, 19, 5583, 37, 261107, 65550, 65624, 58] avg=65536.0000 max=261107 max_rank=4 min=19 min_rank=1 imbalance_ratio=3.984177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] avg=65536.0000 max=66192 max_rank=0 min=63251 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=140050 max_load_reduction_pct=0.679057 moved_to_replicas=205413 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] avg=65536.0000 max=66192 max_rank=0 min=63251 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=140050 max_load_reduction_pct=0.679057 moved_to_replicas=205413 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] avg=65536.0000 max=66192 max_rank=0 min=63934 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=194915 max_load_reduction_pct=0.746495 moved_to_replicas=255033 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65204, 63251, 66192, 66192, 65504, 65561, 66192] avg=65536.0000 max=66192 max_rank=0 min=63251 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=140050 max_load_reduction_pct=0.679057 moved_to_replicas=205413 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[126310, 19, 5583, 37, 261107, 65550, 65624, 58] avg=65536.0000 max=261107 max_rank=4 min=19 min_rank=1 imbalance_ratio=3.984177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] avg=65536.0000 max=66192 max_rank=0 min=63934 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=194915 max_load_reduction_pct=0.746495 moved_to_replicas=255033 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[126310, 19, 5583, 37, 261107, 65550, 65624, 58] avg=65536.0000 max=261107 max_rank=4 min=19 min_rank=1 imbalance_ratio=3.984177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[130190, 196526, 64441, 60381, 19, 3003, 4209, 65519] avg=65536.0000 max=196526 max_rank=1 min=19 min_rank=4 imbalance_ratio=2.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[126310, 19, 5583, 37, 261107, 65550, 65624, 58] avg=65536.0000 max=261107 max_rank=4 min=19 min_rank=1 imbalance_ratio=3.984177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[126310, 19, 5583, 37, 261107, 65550, 65624, 58] avg=65536.0000 max=261107 max_rank=4 min=19 min_rank=1 imbalance_ratio=3.984177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] avg=65536.0000 max=66192 max_rank=0 min=63934 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=194915 max_load_reduction_pct=0.746495 moved_to_replicas=255033 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[130190, 196526, 64441, 60381, 19, 3003, 4209, 65519] avg=65536.0000 max=196526 max_rank=1 min=19 min_rank=4 imbalance_ratio=2.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] avg=65536.0000 max=66192 max_rank=0 min=63934 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=194915 max_load_reduction_pct=0.746495 moved_to_replicas=255033 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[126310, 19, 5583, 37, 261107, 65550, 65624, 58] avg=65536.0000 max=261107 max_rank=4 min=19 min_rank=1 imbalance_ratio=3.984177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] avg=65536.0000 max=66192 max_rank=0 min=64023 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130334 max_load_reduction_pct=0.663190 moved_to_replicas=194332 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[130190, 196526, 64441, 60381, 19, 3003, 4209, 65519] avg=65536.0000 max=196526 max_rank=1 min=19 min_rank=4 imbalance_ratio=2.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] avg=65536.0000 max=66192 max_rank=0 min=63934 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=194915 max_load_reduction_pct=0.746495 moved_to_replicas=255033 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] avg=65536.0000 max=66192 max_rank=0 min=63934 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=194915 max_load_reduction_pct=0.746495 moved_to_replicas=255033 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] avg=65536.0000 max=66192 max_rank=0 min=64023 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130334 max_load_reduction_pct=0.663190 moved_to_replicas=194332 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[66192, 65539, 65701, 65556, 66192, 65550, 65624, 63934] avg=65536.0000 max=66192 max_rank=0 min=63934 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=194915 max_load_reduction_pct=0.746495 moved_to_replicas=255033 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[130190, 196526, 64441, 60381, 19, 3003, 4209, 65519] avg=65536.0000 max=196526 max_rank=1 min=19 min_rank=4 imbalance_ratio=2.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] avg=65536.0000 max=66192 max_rank=0 min=64023 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130334 max_load_reduction_pct=0.663190 moved_to_replicas=194332 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[130190, 196526, 64441, 60381, 19, 3003, 4209, 65519] avg=65536.0000 max=196526 max_rank=1 min=19 min_rank=4 imbalance_ratio=2.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[131194, 124078, 24, 548, 6550, 130832, 9, 131053] avg=65536.0000 max=131194 max_rank=0 min=9 min_rank=6 imbalance_ratio=2.001862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[130190, 196526, 64441, 60381, 19, 3003, 4209, 65519] avg=65536.0000 max=196526 max_rank=1 min=19 min_rank=4 imbalance_ratio=2.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[130190, 196526, 64441, 60381, 19, 3003, 4209, 65519] avg=65536.0000 max=196526 max_rank=1 min=19 min_rank=4 imbalance_ratio=2.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[131194, 124078, 24, 548, 6550, 130832, 9, 131053] avg=65536.0000 max=131194 max_rank=0 min=9 min_rank=6 imbalance_ratio=2.001862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] avg=65536.0000 max=66192 max_rank=0 min=64023 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130334 max_load_reduction_pct=0.663190 moved_to_replicas=194332 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] avg=65536.0000 max=66192 max_rank=0 min=64436 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.495465 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] avg=65536.0000 max=66192 max_rank=0 min=64023 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130334 max_load_reduction_pct=0.663190 moved_to_replicas=194332 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[130190, 196526, 64441, 60381, 19, 3003, 4209, 65519] avg=65536.0000 max=196526 max_rank=1 min=19 min_rank=4 imbalance_ratio=2.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[131194, 124078, 24, 548, 6550, 130832, 9, 131053] avg=65536.0000 max=131194 max_rank=0 min=9 min_rank=6 imbalance_ratio=2.001862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] avg=65536.0000 max=66192 max_rank=0 min=64023 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130334 max_load_reduction_pct=0.663190 moved_to_replicas=194332 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] avg=65536.0000 max=66192 max_rank=0 min=64023 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130334 max_load_reduction_pct=0.663190 moved_to_replicas=194332 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] avg=65536.0000 max=66192 max_rank=0 min=64436 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.495465 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[66192, 66192, 64441, 64023, 65537, 66192, 66192, 65519] avg=65536.0000 max=66192 max_rank=0 min=64023 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130334 max_load_reduction_pct=0.663190 moved_to_replicas=194332 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[131194, 124078, 24, 548, 6550, 130832, 9, 131053] avg=65536.0000 max=131194 max_rank=0 min=9 min_rank=6 imbalance_ratio=2.001862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] avg=65536.0000 max=66192 max_rank=0 min=64436 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.495465 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[196475, 1581, 65516, 123, 146, 194436, 64906, 1105] avg=65536.0000 max=196475 max_rank=0 min=123 min_rank=3 imbalance_ratio=2.997971 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[131194, 124078, 24, 548, 6550, 130832, 9, 131053] avg=65536.0000 max=131194 max_rank=0 min=9 min_rank=6 imbalance_ratio=2.001862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[196475, 1581, 65516, 123, 146, 194436, 64906, 1105] avg=65536.0000 max=196475 max_rank=0 min=123 min_rank=3 imbalance_ratio=2.997971 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[131194, 124078, 24, 548, 6550, 130832, 9, 131053] avg=65536.0000 max=131194 max_rank=0 min=9 min_rank=6 imbalance_ratio=2.001862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[131194, 124078, 24, 548, 6550, 130832, 9, 131053] avg=65536.0000 max=131194 max_rank=0 min=9 min_rank=6 imbalance_ratio=2.001862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] avg=65536.0000 max=66192 max_rank=0 min=64436 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.495465 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] avg=65536.0000 max=66192 max_rank=0 min=64738 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130283 max_load_reduction_pct=0.663102 moved_to_replicas=258527 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] avg=65536.0000 max=66192 max_rank=0 min=64436 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.495465 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[131194, 124078, 24, 548, 6550, 130832, 9, 131053] avg=65536.0000 max=131194 max_rank=0 min=9 min_rank=6 imbalance_ratio=2.001862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[196475, 1581, 65516, 123, 146, 194436, 64906, 1105] avg=65536.0000 max=196475 max_rank=0 min=123 min_rank=3 imbalance_ratio=2.997971 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] avg=65536.0000 max=66192 max_rank=0 min=64738 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130283 max_load_reduction_pct=0.663102 moved_to_replicas=258527 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] avg=65536.0000 max=66192 max_rank=0 min=64436 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.495465 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] avg=65536.0000 max=66192 max_rank=0 min=64436 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.495465 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[66192, 66192, 64885, 65188, 64436, 66192, 65011, 66192] avg=65536.0000 max=66192 max_rank=0 min=64436 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.495465 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[196475, 1581, 65516, 123, 146, 194436, 64906, 1105] avg=65536.0000 max=196475 max_rank=0 min=123 min_rank=3 imbalance_ratio=2.997971 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131008, 65548, 121, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=131008 max_rank=0 min=121 min_rank=2 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] avg=65536.0000 max=66192 max_rank=0 min=64738 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130283 max_load_reduction_pct=0.663102 moved_to_replicas=258527 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[196475, 1581, 65516, 123, 146, 194436, 64906, 1105] avg=65536.0000 max=196475 max_rank=0 min=123 min_rank=3 imbalance_ratio=2.997971 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131008, 65548, 121, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=131008 max_rank=0 min=121 min_rank=2 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[196475, 1581, 65516, 123, 146, 194436, 64906, 1105] avg=65536.0000 max=196475 max_rank=0 min=123 min_rank=3 imbalance_ratio=2.997971 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[196475, 1581, 65516, 123, 146, 194436, 64906, 1105] avg=65536.0000 max=196475 max_rank=0 min=123 min_rank=3 imbalance_ratio=2.997971 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] avg=65536.0000 max=66192 max_rank=0 min=64738 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130283 max_load_reduction_pct=0.663102 moved_to_replicas=258527 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=66192 max_rank=0 min=64937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64816 max_load_reduction_pct=0.494748 moved_to_replicas=64816 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] avg=65536.0000 max=66192 max_rank=0 min=64738 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130283 max_load_reduction_pct=0.663102 moved_to_replicas=258527 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[196475, 1581, 65516, 123, 146, 194436, 64906, 1105] avg=65536.0000 max=196475 max_rank=0 min=123 min_rank=3 imbalance_ratio=2.997971 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131008, 65548, 121, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=131008 max_rank=0 min=121 min_rank=2 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=66192 max_rank=0 min=64937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64816 max_load_reduction_pct=0.494748 moved_to_replicas=64816 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] avg=65536.0000 max=66192 max_rank=0 min=64738 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130283 max_load_reduction_pct=0.663102 moved_to_replicas=258527 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] avg=65536.0000 max=66192 max_rank=0 min=64738 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130283 max_load_reduction_pct=0.663102 moved_to_replicas=258527 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[66192, 64738, 65516, 65618, 64934, 66192, 64906, 66192] avg=65536.0000 max=66192 max_rank=0 min=64738 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130283 max_load_reduction_pct=0.663102 moved_to_replicas=258527 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[63581, 195596, 66884, 131087, 37, 1525, 65578, 0] avg=65536.0000 max=195596 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.984558 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=66192 max_rank=0 min=64937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64816 max_load_reduction_pct=0.494748 moved_to_replicas=64816 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131008, 65548, 121, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=131008 max_rank=0 min=121 min_rank=2 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131008, 65548, 121, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=131008 max_rank=0 min=121 min_rank=2 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[63581, 195596, 66884, 131087, 37, 1525, 65578, 0] avg=65536.0000 max=195596 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.984558 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131008, 65548, 121, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=131008 max_rank=0 min=121 min_rank=2 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131008, 65548, 121, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=131008 max_rank=0 min=121 min_rank=2 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] avg=65536.0000 max=66192 max_rank=1 min=63929 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129404 max_load_reduction_pct=0.661588 moved_to_replicas=194991 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131008, 65548, 121, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=131008 max_rank=0 min=121 min_rank=2 imbalance_ratio=1.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=66192 max_rank=0 min=64937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64816 max_load_reduction_pct=0.494748 moved_to_replicas=64816 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=66192 max_rank=0 min=64937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64816 max_load_reduction_pct=0.494748 moved_to_replicas=64816 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] avg=65536.0000 max=66192 max_rank=1 min=63929 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129404 max_load_reduction_pct=0.661588 moved_to_replicas=194991 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[63581, 195596, 66884, 131087, 37, 1525, 65578, 0] avg=65536.0000 max=195596 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.984558 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=66192 max_rank=0 min=64937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64816 max_load_reduction_pct=0.494748 moved_to_replicas=64816 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=66192 max_rank=0 min=64937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64816 max_load_reduction_pct=0.494748 moved_to_replicas=64816 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[66192, 65548, 64937, 65405, 65524, 65521, 65568, 65593] avg=65536.0000 max=66192 max_rank=0 min=64937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64816 max_load_reduction_pct=0.494748 moved_to_replicas=64816 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[218, 65570, 196371, 66100, 44, 64923, 65550, 65512] avg=65536.0000 max=196371 max_rank=2 min=44 min_rank=4 imbalance_ratio=2.996384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] avg=65536.0000 max=66192 max_rank=1 min=63929 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129404 max_load_reduction_pct=0.661588 moved_to_replicas=194991 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[63581, 195596, 66884, 131087, 37, 1525, 65578, 0] avg=65536.0000 max=195596 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.984558 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[218, 65570, 196371, 66100, 44, 64923, 65550, 65512] avg=65536.0000 max=196371 max_rank=2 min=44 min_rank=4 imbalance_ratio=2.996384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[63581, 195596, 66884, 131087, 37, 1525, 65578, 0] avg=65536.0000 max=195596 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.984558 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64903 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130179 max_load_reduction_pct=0.662924 moved_to_replicas=130179 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[63581, 195596, 66884, 131087, 37, 1525, 65578, 0] avg=65536.0000 max=195596 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.984558 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[63581, 195596, 66884, 131087, 37, 1525, 65578, 0] avg=65536.0000 max=195596 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.984558 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[63581, 195596, 66884, 131087, 37, 1525, 65578, 0] avg=65536.0000 max=195596 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.984558 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64903 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130179 max_load_reduction_pct=0.662924 moved_to_replicas=130179 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] avg=65536.0000 max=66192 max_rank=1 min=63929 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129404 max_load_reduction_pct=0.661588 moved_to_replicas=194991 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] avg=65536.0000 max=66192 max_rank=1 min=63929 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129404 max_load_reduction_pct=0.661588 moved_to_replicas=194991 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[218, 65570, 196371, 66100, 44, 64923, 65550, 65512] avg=65536.0000 max=196371 max_rank=2 min=44 min_rank=4 imbalance_ratio=2.996384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] avg=65536.0000 max=66192 max_rank=1 min=63929 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129404 max_load_reduction_pct=0.661588 moved_to_replicas=194991 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] avg=65536.0000 max=66192 max_rank=1 min=63929 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129404 max_load_reduction_pct=0.661588 moved_to_replicas=194991 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[64501, 66192, 66192, 66192, 63929, 66192, 65578, 65512] avg=65536.0000 max=66192 max_rank=1 min=63929 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=129404 max_load_reduction_pct=0.661588 moved_to_replicas=194991 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64903 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130179 max_load_reduction_pct=0.662924 moved_to_replicas=130179 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[218, 65570, 196371, 66100, 44, 64923, 65550, 65512] avg=65536.0000 max=196371 max_rank=2 min=44 min_rank=4 imbalance_ratio=2.996384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[218, 65570, 196371, 66100, 44, 64923, 65550, 65512] avg=65536.0000 max=196371 max_rank=2 min=44 min_rank=4 imbalance_ratio=2.996384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[218, 65570, 196371, 66100, 44, 64923, 65550, 65512] avg=65536.0000 max=196371 max_rank=2 min=44 min_rank=4 imbalance_ratio=2.996384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[218, 65570, 196371, 66100, 44, 64923, 65550, 65512] avg=65536.0000 max=196371 max_rank=2 min=44 min_rank=4 imbalance_ratio=2.996384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[218, 65570, 196371, 66100, 44, 64923, 65550, 65512] avg=65536.0000 max=196371 max_rank=2 min=44 min_rank=4 imbalance_ratio=2.996384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64903 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130179 max_load_reduction_pct=0.662924 moved_to_replicas=130179 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64903 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130179 max_load_reduction_pct=0.662924 moved_to_replicas=130179 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64903 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130179 max_load_reduction_pct=0.662924 moved_to_replicas=130179 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64903 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130179 max_load_reduction_pct=0.662924 moved_to_replicas=130179 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[64903, 65570, 66192, 66100, 65538, 64923, 65550, 65512] avg=65536.0000 max=66192 max_rank=2 min=64903 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130179 max_load_reduction_pct=0.662924 moved_to_replicas=130179 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=12 total_expected_recv=782626 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193792 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=12 total_expected_recv=783007 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193408 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=12 total_expected_recv=788409 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187136 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=12 total_expected_recv=789296 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186112 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=12 total_expected_recv=784974 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190592 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=12 total_expected_recv=786461 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189312 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=12 total_expected_recv=788563 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186496 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=12 total_expected_recv=788120 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188032 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 15:44:41] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 342.6 | throughput per GPU (TFLOP/s/GPU): 527.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.026621E+01 | loss scale: 1.0 | grad norm: 6.286 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65903 expected_overflow=0 exact_recv_per_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=81920 expected_recv=61235 expected_overflow=0 exact_recv_per_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65903 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64886 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64974 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64417 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=81920 expected_recv=61235 actual_valid_recv=62720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62720 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64886 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64886 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64895 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65353 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65122 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64895 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65353 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65456 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65122 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64974 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65905 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=81920 expected_recv=64417 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65905 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65578 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65636 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=81920 expected_recv=64873 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65160 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65578 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65636 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=81920 expected_recv=64873 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65160 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65116 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65116 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65297 expected_overflow=0 exact_recv_per_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=81920 expected_recv=64868 expected_overflow=0 exact_recv_per_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65297 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65589 actual_valid_recv=65792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=81920 expected_recv=64419 expected_overflow=0 exact_recv_per_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=81920 expected_recv=64868 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=81920 expected_recv=64419 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64861 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64893 expected_overflow=0 exact_recv_per_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64903 expected_overflow=0 exact_recv_per_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65506 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64893 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64903 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64861 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64886 expected_overflow=0 exact_recv_per_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=81920 expected_recv=64886 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65575 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65520 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=81920 expected_recv=62887 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=81920 expected_recv=62887 actual_valid_recv=63616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=81920 expected_recv=64333 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=81920 expected_recv=63326 expected_overflow=0 exact_recv_per_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65567 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=81920 expected_recv=63326 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65763 expected_overflow=0 exact_recv_per_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65763 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65748 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=81920 expected_recv=64333 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65748 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=81920 expected_recv=62917 expected_overflow=0 exact_recv_per_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65441 expected_overflow=0 exact_recv_per_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=81920 expected_recv=62917 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65441 actual_valid_recv=65664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65199 expected_overflow=0 exact_recv_per_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65551 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65267 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65199 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65267 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65547 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65399 expected_overflow=0 exact_recv_per_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65399 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65114 expected_overflow=0 exact_recv_per_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65498 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65528 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65235 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65114 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65585 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65481 expected_overflow=0 exact_recv_per_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65235 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65955 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65481 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=81920 expected_recv=63714 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65955 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=81920 expected_recv=63714 actual_valid_recv=64000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65528 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=81920 expected_recv=64178 expected_overflow=0 exact_recv_per_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=81920 expected_recv=64178 actual_valid_recv=64384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65082 expected_overflow=0 exact_recv_per_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65494 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65082 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65620 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65468 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65000 expected_overflow=0 exact_recv_per_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65000 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65245 expected_overflow=0 exact_recv_per_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65245 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64147 expected_overflow=0 exact_recv_per_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64147 actual_valid_recv=64768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=81920 expected_recv=64566 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65264 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=81920 expected_recv=64566 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65264 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65552 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65180 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64957 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=65664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65180 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64957 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64506 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=81920 expected_recv=64506 actual_valid_recv=64896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64862 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64916 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64862 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65430 expected_overflow=0 exact_recv_per_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65430 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65002 expected_overflow=0 exact_recv_per_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64956 expected_overflow=0 exact_recv_per_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65002 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64956 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65604 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64916 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64153 expected_overflow=0 exact_recv_per_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64870 expected_overflow=0 exact_recv_per_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65358 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65682 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64870 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65358 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64995 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65682 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64995 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64807 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64153 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64875 expected_overflow=0 exact_recv_per_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=81920 expected_recv=64875 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64971 expected_overflow=0 exact_recv_per_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64495 expected_overflow=0 exact_recv_per_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64971 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64495 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66048 expected_overflow=0 exact_recv_per_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66048 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64807 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65326 expected_overflow=0 exact_recv_per_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[61235, 114066, 75413, 46334, 136691, 16176, 38513, 35860] avg=65536.0000 max=136691 max_rank=4 min=16176 min_rank=5 imbalance_ratio=2.085739 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65326 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[61235, 114066, 75413, 46334, 136691, 16176, 38513, 35860] avg=65536.0000 max=136691 max_rank=4 min=16176 min_rank=5 imbalance_ratio=2.085739 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[61235, 114066, 75413, 46334, 136691, 16176, 38513, 35860] avg=65536.0000 max=136691 max_rank=4 min=16176 min_rank=5 imbalance_ratio=2.085739 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[61235, 114066, 75413, 46334, 136691, 16176, 38513, 35860] avg=65536.0000 max=136691 max_rank=4 min=16176 min_rank=5 imbalance_ratio=2.085739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[61235, 114066, 75413, 46334, 136691, 16176, 38513, 35860] avg=65536.0000 max=136691 max_rank=4 min=16176 min_rank=5 imbalance_ratio=2.085739 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[60345, 114117, 75993, 38960, 136473, 16903, 42572, 38925] avg=65536.0000 max=136473 max_rank=4 min=16903 min_rank=5 imbalance_ratio=2.082413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[60944, 66191, 66192, 66192, 66192, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=60944 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70280 max_load_reduction_pct=0.514974 moved_to_replicas=128007 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[131163, 130988, 1109, 131070, 64417, 8, 3, 65530] avg=65536.0000 max=131163 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.001389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65905, 66192, 64417, 64886, 64974, 65530] avg=65536.0000 max=66192 max_rank=0 min=64417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64971 max_load_reduction_pct=0.495345 moved_to_replicas=194645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[64644, 65578, 132194, 106, 10, 10, 65160, 196586] avg=65536.0000 max=196586 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[65116, 65578, 66192, 65636, 65541, 64873, 65160, 66192] avg=65536.0000 max=66192 max_rank=2 min=64873 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130394 max_load_reduction_pct=0.663292 moved_to_replicas=196396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[2, 26, 12, 65534, 161336, 5, 35247, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[65532, 65555, 64886, 65534, 66193, 65535, 64861, 66192] avg=65536.0000 max=66193 max_rank=4 min=64861 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=195933 max_load_reduction_pct=0.747476 moved_to_replicas=291078 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[215173, 7258, 40488, 103, 195658, 5, 65533, 70] avg=65536.0000 max=215173 max_rank=0 min=5 min_rank=5 imbalance_ratio=3.283279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[66193, 66192, 62887, 66193, 66192, 65523, 65533, 65575] avg=65536.0000 max=66193 max_rank=0 min=62887 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=148980 max_load_reduction_pct=0.692373 moved_to_replicas=278447 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65551, 6, 65199, 220, 191106, 29, 136640, 65537] avg=65536.0000 max=191106 max_rank=4 min=6 min_rank=1 imbalance_ratio=2.916046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65551, 65536, 65199, 65748, 66192, 64333, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=64333 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124914 max_load_reduction_pct=0.653637 moved_to_replicas=195362 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[40, 196917, 65481, 65547, 65528, 65267, 10, 65498] avg=65536.0000 max=196917 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.004715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65235, 66192, 65481, 65547, 65528, 65267, 65540, 65498] avg=65536.0000 max=66192 max_rank=1 min=65235 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130725 max_load_reduction_pct=0.663858 moved_to_replicas=130725 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[259757, 62804, 99, 4588, 65536, 65955, 3, 65546] avg=65536.0000 max=259757 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.963577 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[66192, 63714, 65620, 66192, 65536, 65955, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=63714 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=193565 max_load_reduction_pct=0.745177 moved_to_replicas=193565 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 130588, 15, 65468, 65264, 66347] avg=65536.0000 max=130588 max_rank=3 min=15 min_rank=4 imbalance_ratio=1.992615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65552, 65525, 65529, 66192, 64566, 65468, 65264, 66192] avg=65536.0000 max=66192 max_rank=3 min=64566 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64396 max_load_reduction_pct=0.493123 moved_to_replicas=64551 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65539, 131071, 65530, 78, 63096, 65180, 2751, 131043] avg=65536.0000 max=131071 max_rank=1 min=78 min_rank=3 imbalance_ratio=1.999985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65539, 66192, 65530, 64957, 64506, 65180, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64506 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64879 max_load_reduction_pct=0.494991 moved_to_replicas=129730 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[4, 196575, 79, 130999, 5, 196512, 109, 5] avg=65536.0000 max=196575 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.999496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65530, 66192, 64870, 66192, 64862, 66192, 64916, 65534] avg=65536.0000 max=66192 max_rank=1 min=64862 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130383 max_load_reduction_pct=0.663274 moved_to_replicas=325510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[131183, 65682, 65526, 65358, 122004, 4, 8997, 65534] avg=65536.0000 max=131183 max_rank=0 min=4 min_rank=5 imbalance_ratio=2.001694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[66192, 65682, 65526, 65358, 66194, 64995, 64807, 65534] avg=65536.0000 max=66194 max_rank=4 min=64807 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64989 max_load_reduction_pct=0.495407 moved_to_replicas=120803 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=12 total_expected_recv=784399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192640 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=12 total_expected_recv=788741 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187136 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=12 total_expected_recv=789933 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187136 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=12 total_expected_recv=785919 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192384 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=12 total_expected_recv=789714 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187264 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=12 total_expected_recv=785165 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192000 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=12 total_expected_recv=783768 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192896 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=12 total_expected_recv=783817 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194304 peak_demand=66192 required_capacity=1.010010 - [2026-08-26 14:59:20] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 339.6 | throughput per GPU (TFLOP/s/GPU): 532.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.796568E+00 | loss scale: 1.0 | grad norm: 5.584 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] avg=65536.0000 max=66193 max_rank=3 min=61235 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70498 max_load_reduction_pct=0.515747 moved_to_replicas=127594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[61235, 114066, 75413, 46334, 136691, 16176, 38513, 35860] avg=65536.0000 max=136691 max_rank=4 min=16176 min_rank=5 imbalance_ratio=2.085739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] avg=65536.0000 max=66193 max_rank=3 min=61235 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70498 max_load_reduction_pct=0.515747 moved_to_replicas=127594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] avg=65536.0000 max=66193 max_rank=3 min=61235 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70498 max_load_reduction_pct=0.515747 moved_to_replicas=127594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] avg=65536.0000 max=66193 max_rank=3 min=61235 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70498 max_load_reduction_pct=0.515747 moved_to_replicas=127594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] avg=65536.0000 max=66193 max_rank=3 min=61235 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70498 max_load_reduction_pct=0.515747 moved_to_replicas=127594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[61235, 114066, 75413, 46334, 136691, 16176, 38513, 35860] avg=65536.0000 max=136691 max_rank=4 min=16176 min_rank=5 imbalance_ratio=2.085739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] avg=65536.0000 max=66193 max_rank=3 min=61235 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70498 max_load_reduction_pct=0.515747 moved_to_replicas=127594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[131084, 130493, 65456, 131070, 821, 8, 3, 65353] avg=65536.0000 max=131084 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[131084, 130493, 65456, 131070, 821, 8, 3, 65353] avg=65536.0000 max=131084 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[61235, 114066, 75413, 46334, 136691, 16176, 38513, 35860] avg=65536.0000 max=136691 max_rank=4 min=16176 min_rank=5 imbalance_ratio=2.085739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[131084, 130493, 65456, 131070, 821, 8, 3, 65353] avg=65536.0000 max=131084 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[131084, 130493, 65456, 131070, 821, 8, 3, 65353] avg=65536.0000 max=131084 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] avg=65536.0000 max=66193 max_rank=3 min=61235 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70498 max_load_reduction_pct=0.515747 moved_to_replicas=127594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] avg=65536.0000 max=66192 max_rank=0 min=64886 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64892 max_load_reduction_pct=0.495041 moved_to_replicas=194071 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] avg=65536.0000 max=66192 max_rank=0 min=64886 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64892 max_load_reduction_pct=0.495041 moved_to_replicas=194071 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[61235, 66191, 66191, 66193, 66192, 66191, 66192, 65903] avg=65536.0000 max=66193 max_rank=3 min=61235 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=70498 max_load_reduction_pct=0.515747 moved_to_replicas=127594 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[131084, 130493, 65456, 131070, 821, 8, 3, 65353] avg=65536.0000 max=131084 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[131084, 130493, 65456, 131070, 821, 8, 3, 65353] avg=65536.0000 max=131084 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] avg=65536.0000 max=66192 max_rank=0 min=64886 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64892 max_load_reduction_pct=0.495041 moved_to_replicas=194071 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] avg=65536.0000 max=66192 max_rank=0 min=64886 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64892 max_load_reduction_pct=0.495041 moved_to_replicas=194071 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] avg=65536.0000 max=66192 max_rank=0 min=64886 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64892 max_load_reduction_pct=0.495041 moved_to_replicas=194071 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] avg=65536.0000 max=66192 max_rank=0 min=64886 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64892 max_load_reduction_pct=0.495041 moved_to_replicas=194071 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[131084, 130493, 65456, 131070, 821, 8, 3, 65353] avg=65536.0000 max=131084 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[24604, 65589, 133379, 38821, 10, 7, 65297, 196581] avg=65536.0000 max=196581 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.999588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[24604, 65589, 133379, 38821, 10, 7, 65297, 196581] avg=65536.0000 max=196581 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.999588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[131084, 130493, 65456, 131070, 821, 8, 3, 65353] avg=65536.0000 max=131084 max_rank=0 min=3 min_rank=6 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[24604, 65589, 133379, 38821, 10, 7, 65297, 196581] avg=65536.0000 max=196581 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.999588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[24604, 65589, 133379, 38821, 10, 7, 65297, 196581] avg=65536.0000 max=196581 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.999588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] avg=65536.0000 max=66192 max_rank=0 min=64886 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64892 max_load_reduction_pct=0.495041 moved_to_replicas=194071 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] avg=65536.0000 max=66193 max_rank=0 min=64419 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=130388 max_load_reduction_pct=0.663279 moved_to_replicas=197576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] avg=65536.0000 max=66193 max_rank=0 min=64419 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=130388 max_load_reduction_pct=0.663279 moved_to_replicas=197576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[66192, 66192, 65456, 66192, 65122, 64886, 64895, 65353] avg=65536.0000 max=66192 max_rank=0 min=64886 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64892 max_load_reduction_pct=0.495041 moved_to_replicas=194071 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] avg=65536.0000 max=66193 max_rank=0 min=64419 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=130388 max_load_reduction_pct=0.663279 moved_to_replicas=197576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[24604, 65589, 133379, 38821, 10, 7, 65297, 196581] avg=65536.0000 max=196581 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.999588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[24604, 65589, 133379, 38821, 10, 7, 65297, 196581] avg=65536.0000 max=196581 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.999588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] avg=65536.0000 max=66193 max_rank=0 min=64419 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=130388 max_load_reduction_pct=0.663279 moved_to_replicas=197576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] avg=65536.0000 max=66193 max_rank=0 min=64419 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=130388 max_load_reduction_pct=0.663279 moved_to_replicas=197576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] avg=65536.0000 max=66193 max_rank=0 min=64419 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=130388 max_load_reduction_pct=0.663279 moved_to_replicas=197576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[24604, 65589, 133379, 38821, 10, 7, 65297, 196581] avg=65536.0000 max=196581 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.999588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[2, 35, 19, 65535, 131060, 5, 65506, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[2, 35, 19, 65535, 131060, 5, 65506, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[2, 35, 19, 65535, 131060, 5, 65506, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[24604, 65589, 133379, 38821, 10, 7, 65297, 196581] avg=65536.0000 max=196581 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.999588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[2, 35, 19, 65535, 131060, 5, 65506, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] avg=65536.0000 max=66193 max_rank=0 min=64419 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=130388 max_load_reduction_pct=0.663279 moved_to_replicas=197576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] avg=65536.0000 max=66192 max_rank=4 min=64893 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195934 max_load_reduction_pct=0.747480 moved_to_replicas=260802 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] avg=65536.0000 max=66192 max_rank=4 min=64893 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195934 max_load_reduction_pct=0.747480 moved_to_replicas=260802 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] avg=65536.0000 max=66192 max_rank=4 min=64893 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195934 max_load_reduction_pct=0.747480 moved_to_replicas=260802 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[66193, 65589, 66192, 64419, 64868, 65538, 65297, 66192] avg=65536.0000 max=66193 max_rank=0 min=64419 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=130388 max_load_reduction_pct=0.663279 moved_to_replicas=197576 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[2, 35, 19, 65535, 131060, 5, 65506, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[2, 35, 19, 65535, 131060, 5, 65506, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] avg=65536.0000 max=66192 max_rank=4 min=64893 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195934 max_load_reduction_pct=0.747480 moved_to_replicas=260802 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] avg=65536.0000 max=66192 max_rank=4 min=64893 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195934 max_load_reduction_pct=0.747480 moved_to_replicas=260802 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] avg=65536.0000 max=66192 max_rank=4 min=64893 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195934 max_load_reduction_pct=0.747480 moved_to_replicas=260802 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[259401, 52, 4162, 233, 194836, 6, 65520, 78] avg=65536.0000 max=259401 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.958145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[2, 35, 19, 65535, 131060, 5, 65506, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[259401, 52, 4162, 233, 194836, 6, 65520, 78] avg=65536.0000 max=259401 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.958145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[259401, 52, 4162, 233, 194836, 6, 65520, 78] avg=65536.0000 max=259401 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.958145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[2, 35, 19, 65535, 131060, 5, 65506, 262126] avg=65536.0000 max=262126 max_rank=7 min=2 min_rank=0 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[259401, 52, 4162, 233, 194836, 6, 65520, 78] avg=65536.0000 max=259401 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.958145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] avg=65536.0000 max=66192 max_rank=0 min=63326 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=193209 max_load_reduction_pct=0.744828 moved_to_replicas=321853 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] avg=65536.0000 max=66192 max_rank=4 min=64893 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195934 max_load_reduction_pct=0.747480 moved_to_replicas=260802 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] avg=65536.0000 max=66192 max_rank=0 min=63326 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=193209 max_load_reduction_pct=0.744828 moved_to_replicas=321853 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] avg=65536.0000 max=66192 max_rank=0 min=63326 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=193209 max_load_reduction_pct=0.744828 moved_to_replicas=321853 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[259401, 52, 4162, 233, 194836, 6, 65520, 78] avg=65536.0000 max=259401 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.958145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[65532, 64903, 64893, 65535, 66192, 65535, 65506, 66192] avg=65536.0000 max=66192 max_rank=4 min=64893 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195934 max_load_reduction_pct=0.747480 moved_to_replicas=260802 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] avg=65536.0000 max=66192 max_rank=0 min=63326 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=193209 max_load_reduction_pct=0.744828 moved_to_replicas=321853 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[259401, 52, 4162, 233, 194836, 6, 65520, 78] avg=65536.0000 max=259401 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.958145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] avg=65536.0000 max=66192 max_rank=0 min=63326 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=193209 max_load_reduction_pct=0.744828 moved_to_replicas=321853 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65534, 5, 113944, 104, 150737, 26, 128404, 65534] avg=65536.0000 max=150737 max_rank=4 min=5 min_rank=1 imbalance_ratio=2.300064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] avg=65536.0000 max=66192 max_rank=0 min=63326 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=193209 max_load_reduction_pct=0.744828 moved_to_replicas=321853 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[259401, 52, 4162, 233, 194836, 6, 65520, 78] avg=65536.0000 max=259401 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.958145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65534, 5, 113944, 104, 150737, 26, 128404, 65534] avg=65536.0000 max=150737 max_rank=4 min=5 min_rank=1 imbalance_ratio=2.300064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65534, 5, 113944, 104, 150737, 26, 128404, 65534] avg=65536.0000 max=150737 max_rank=4 min=5 min_rank=1 imbalance_ratio=2.300064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[259401, 52, 4162, 233, 194836, 6, 65520, 78] avg=65536.0000 max=259401 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.958145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65534, 5, 113944, 104, 150737, 26, 128404, 65534] avg=65536.0000 max=150737 max_rank=4 min=5 min_rank=1 imbalance_ratio=2.300064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] avg=65536.0000 max=66193 max_rank=2 min=62917 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=84544 max_load_reduction_pct=0.560871 moved_to_replicas=194509 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] avg=65536.0000 max=66193 max_rank=2 min=62917 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=84544 max_load_reduction_pct=0.560871 moved_to_replicas=194509 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] avg=65536.0000 max=66192 max_rank=0 min=63326 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=193209 max_load_reduction_pct=0.744828 moved_to_replicas=321853 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[66192, 65567, 66192, 65763, 66192, 65536, 65520, 63326] avg=65536.0000 max=66192 max_rank=0 min=63326 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=193209 max_load_reduction_pct=0.744828 moved_to_replicas=321853 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] avg=65536.0000 max=66193 max_rank=2 min=62917 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=84544 max_load_reduction_pct=0.560871 moved_to_replicas=194509 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65534, 5, 113944, 104, 150737, 26, 128404, 65534] avg=65536.0000 max=150737 max_rank=4 min=5 min_rank=1 imbalance_ratio=2.300064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65534, 5, 113944, 104, 150737, 26, 128404, 65534] avg=65536.0000 max=150737 max_rank=4 min=5 min_rank=1 imbalance_ratio=2.300064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] avg=65536.0000 max=66193 max_rank=2 min=62917 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=84544 max_load_reduction_pct=0.560871 moved_to_replicas=194509 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[23, 196714, 65399, 122, 65523, 65441, 65531, 65535] avg=65536.0000 max=196714 max_rank=1 min=23 min_rank=0 imbalance_ratio=3.001617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[23, 196714, 65399, 122, 65523, 65441, 65531, 65535] avg=65536.0000 max=196714 max_rank=1 min=23 min_rank=0 imbalance_ratio=3.001617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] avg=65536.0000 max=66193 max_rank=2 min=62917 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=84544 max_load_reduction_pct=0.560871 moved_to_replicas=194509 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] avg=65536.0000 max=66193 max_rank=2 min=62917 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=84544 max_load_reduction_pct=0.560871 moved_to_replicas=194509 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65534, 5, 113944, 104, 150737, 26, 128404, 65534] avg=65536.0000 max=150737 max_rank=4 min=5 min_rank=1 imbalance_ratio=2.300064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65534, 5, 113944, 104, 150737, 26, 128404, 65534] avg=65536.0000 max=150737 max_rank=4 min=5 min_rank=1 imbalance_ratio=2.300064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[23, 196714, 65399, 122, 65523, 65441, 65531, 65535] avg=65536.0000 max=196714 max_rank=1 min=23 min_rank=0 imbalance_ratio=3.001617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[23, 196714, 65399, 122, 65523, 65441, 65531, 65535] avg=65536.0000 max=196714 max_rank=1 min=23 min_rank=0 imbalance_ratio=3.001617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] avg=65536.0000 max=66192 max_rank=1 min=65114 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130522 max_load_reduction_pct=0.663511 moved_to_replicas=130522 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] avg=65536.0000 max=66192 max_rank=1 min=65114 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130522 max_load_reduction_pct=0.663511 moved_to_replicas=130522 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] avg=65536.0000 max=66193 max_rank=2 min=62917 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=84544 max_load_reduction_pct=0.560871 moved_to_replicas=194509 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65534, 65535, 66193, 62917, 66192, 66191, 66192, 65534] avg=65536.0000 max=66193 max_rank=2 min=62917 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=84544 max_load_reduction_pct=0.560871 moved_to_replicas=194509 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] avg=65536.0000 max=66192 max_rank=1 min=65114 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130522 max_load_reduction_pct=0.663511 moved_to_replicas=130522 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] avg=65536.0000 max=66192 max_rank=1 min=65114 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130522 max_load_reduction_pct=0.663511 moved_to_replicas=130522 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[23, 196714, 65399, 122, 65523, 65441, 65531, 65535] avg=65536.0000 max=196714 max_rank=1 min=23 min_rank=0 imbalance_ratio=3.001617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[23, 196714, 65399, 122, 65523, 65441, 65531, 65535] avg=65536.0000 max=196714 max_rank=1 min=23 min_rank=0 imbalance_ratio=3.001617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[260537, 60514, 11, 6558, 65534, 65585, 3, 65546] avg=65536.0000 max=260537 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.975479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[260537, 60514, 11, 6558, 65534, 65585, 3, 65546] avg=65536.0000 max=260537 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.975479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] avg=65536.0000 max=66192 max_rank=1 min=65114 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130522 max_load_reduction_pct=0.663511 moved_to_replicas=130522 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] avg=65536.0000 max=66192 max_rank=1 min=65114 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130522 max_load_reduction_pct=0.663511 moved_to_replicas=130522 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[23, 196714, 65399, 122, 65523, 65441, 65531, 65535] avg=65536.0000 max=196714 max_rank=1 min=23 min_rank=0 imbalance_ratio=3.001617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[23, 196714, 65399, 122, 65523, 65441, 65531, 65535] avg=65536.0000 max=196714 max_rank=1 min=23 min_rank=0 imbalance_ratio=3.001617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[260537, 60514, 11, 6558, 65534, 65585, 3, 65546] avg=65536.0000 max=260537 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.975479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[260537, 60514, 11, 6558, 65534, 65585, 3, 65546] avg=65536.0000 max=260537 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.975479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=64178 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194345 max_load_reduction_pct=0.745940 moved_to_replicas=194345 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=64178 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194345 max_load_reduction_pct=0.745940 moved_to_replicas=194345 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] avg=65536.0000 max=66192 max_rank=1 min=65114 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130522 max_load_reduction_pct=0.663511 moved_to_replicas=130522 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65553, 66192, 65399, 65114, 65523, 65441, 65531, 65535] avg=65536.0000 max=66192 max_rank=1 min=65114 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130522 max_load_reduction_pct=0.663511 moved_to_replicas=130522 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=64178 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194345 max_load_reduction_pct=0.745940 moved_to_replicas=194345 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=64178 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194345 max_load_reduction_pct=0.745940 moved_to_replicas=194345 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[260537, 60514, 11, 6558, 65534, 65585, 3, 65546] avg=65536.0000 max=260537 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.975479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[260537, 60514, 11, 6558, 65534, 65585, 3, 65546] avg=65536.0000 max=260537 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.975479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 126855, 30, 65494, 65082, 70744] avg=65536.0000 max=126855 max_rank=3 min=30 min_rank=4 imbalance_ratio=1.935654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 126855, 30, 65494, 65082, 70744] avg=65536.0000 max=126855 max_rank=3 min=30 min_rank=4 imbalance_ratio=1.935654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=64178 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194345 max_load_reduction_pct=0.745940 moved_to_replicas=194345 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=64178 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194345 max_load_reduction_pct=0.745940 moved_to_replicas=194345 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[260537, 60514, 11, 6558, 65534, 65585, 3, 65546] avg=65536.0000 max=260537 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.975479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 126855, 30, 65494, 65082, 70744] avg=65536.0000 max=126855 max_rank=3 min=30 min_rank=4 imbalance_ratio=1.935654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[260537, 60514, 11, 6558, 65534, 65585, 3, 65546] avg=65536.0000 max=260537 max_rank=0 min=3 min_rank=6 imbalance_ratio=3.975479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 126855, 30, 65494, 65082, 70744] avg=65536.0000 max=126855 max_rank=3 min=30 min_rank=4 imbalance_ratio=1.935654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] avg=65536.0000 max=66192 max_rank=3 min=65000 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=60663 max_load_reduction_pct=0.478207 moved_to_replicas=65215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] avg=65536.0000 max=66192 max_rank=3 min=65000 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=60663 max_load_reduction_pct=0.478207 moved_to_replicas=65215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=64178 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194345 max_load_reduction_pct=0.745940 moved_to_replicas=194345 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] avg=65536.0000 max=66192 max_rank=3 min=65000 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=60663 max_load_reduction_pct=0.478207 moved_to_replicas=65215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[66192, 64178, 65528, 66192, 65534, 65585, 65533, 65546] avg=65536.0000 max=66192 max_rank=0 min=64178 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=194345 max_load_reduction_pct=0.745940 moved_to_replicas=194345 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 126855, 30, 65494, 65082, 70744] avg=65536.0000 max=126855 max_rank=3 min=30 min_rank=4 imbalance_ratio=1.935654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] avg=65536.0000 max=66192 max_rank=3 min=65000 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=60663 max_load_reduction_pct=0.478207 moved_to_replicas=65215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 126855, 30, 65494, 65082, 70744] avg=65536.0000 max=126855 max_rank=3 min=30 min_rank=4 imbalance_ratio=1.935654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65604, 131070, 65529, 124, 65430, 2017, 63471, 131043] avg=65536.0000 max=131070 max_rank=1 min=124 min_rank=3 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65604, 131070, 65529, 124, 65430, 2017, 63471, 131043] avg=65536.0000 max=131070 max_rank=1 min=124 min_rank=3 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] avg=65536.0000 max=66192 max_rank=3 min=65000 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=60663 max_load_reduction_pct=0.478207 moved_to_replicas=65215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 126855, 30, 65494, 65082, 70744] avg=65536.0000 max=126855 max_rank=3 min=30 min_rank=4 imbalance_ratio=1.935654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] avg=65536.0000 max=66192 max_rank=3 min=65000 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=60663 max_load_reduction_pct=0.478207 moved_to_replicas=65215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65604, 131070, 65529, 124, 65430, 2017, 63471, 131043] avg=65536.0000 max=131070 max_rank=1 min=124 min_rank=3 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] avg=65536.0000 max=66192 max_rank=1 min=64147 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64878 max_load_reduction_pct=0.494987 moved_to_replicas=129729 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] avg=65536.0000 max=66192 max_rank=1 min=64147 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64878 max_load_reduction_pct=0.494987 moved_to_replicas=129729 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 126855, 30, 65494, 65082, 70744] avg=65536.0000 max=126855 max_rank=3 min=30 min_rank=4 imbalance_ratio=1.935654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65604, 131070, 65529, 124, 65430, 2017, 63471, 131043] avg=65536.0000 max=131070 max_rank=1 min=124 min_rank=3 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] avg=65536.0000 max=66192 max_rank=3 min=65000 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=60663 max_load_reduction_pct=0.478207 moved_to_replicas=65215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] avg=65536.0000 max=66192 max_rank=1 min=64147 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64878 max_load_reduction_pct=0.494987 moved_to_replicas=129729 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65554, 65000, 65529, 66192, 65245, 65494, 65082, 66192] avg=65536.0000 max=66192 max_rank=3 min=65000 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=60663 max_load_reduction_pct=0.478207 moved_to_replicas=65215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65604, 131070, 65529, 124, 65430, 2017, 63471, 131043] avg=65536.0000 max=131070 max_rank=1 min=124 min_rank=3 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65604, 131070, 65529, 124, 65430, 2017, 63471, 131043] avg=65536.0000 max=131070 max_rank=1 min=124 min_rank=3 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] avg=65536.0000 max=66192 max_rank=1 min=64147 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64878 max_load_reduction_pct=0.494987 moved_to_replicas=129729 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[4, 131061, 90055, 131007, 6, 172009, 141, 5] avg=65536.0000 max=172009 max_rank=5 min=4 min_rank=0 imbalance_ratio=2.624649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[4, 131061, 90055, 131007, 6, 172009, 141, 5] avg=65536.0000 max=172009 max_rank=5 min=4 min_rank=0 imbalance_ratio=2.624649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] avg=65536.0000 max=66192 max_rank=1 min=64147 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64878 max_load_reduction_pct=0.494987 moved_to_replicas=129729 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] avg=65536.0000 max=66192 max_rank=1 min=64147 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64878 max_load_reduction_pct=0.494987 moved_to_replicas=129729 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] avg=65536.0000 max=66194 max_rank=2 min=64153 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=105815 max_load_reduction_pct=0.615171 moved_to_replicas=259364 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] avg=65536.0000 max=66194 max_rank=2 min=64153 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=105815 max_load_reduction_pct=0.615171 moved_to_replicas=259364 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[4, 131061, 90055, 131007, 6, 172009, 141, 5] avg=65536.0000 max=172009 max_rank=5 min=4 min_rank=0 imbalance_ratio=2.624649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65604, 131070, 65529, 124, 65430, 2017, 63471, 131043] avg=65536.0000 max=131070 max_rank=1 min=124 min_rank=3 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65604, 131070, 65529, 124, 65430, 2017, 63471, 131043] avg=65536.0000 max=131070 max_rank=1 min=124 min_rank=3 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[4, 131061, 90055, 131007, 6, 172009, 141, 5] avg=65536.0000 max=172009 max_rank=5 min=4 min_rank=0 imbalance_ratio=2.624649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[4, 131061, 90055, 131007, 6, 172009, 141, 5] avg=65536.0000 max=172009 max_rank=5 min=4 min_rank=0 imbalance_ratio=2.624649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] avg=65536.0000 max=66194 max_rank=2 min=64153 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=105815 max_load_reduction_pct=0.615171 moved_to_replicas=259364 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[4, 131061, 90055, 131007, 6, 172009, 141, 5] avg=65536.0000 max=172009 max_rank=5 min=4 min_rank=0 imbalance_ratio=2.624649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] avg=65536.0000 max=66192 max_rank=1 min=64147 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64878 max_load_reduction_pct=0.494987 moved_to_replicas=129729 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65604, 66192, 65529, 65002, 65430, 66192, 64147, 66192] avg=65536.0000 max=66192 max_rank=1 min=64147 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64878 max_load_reduction_pct=0.494987 moved_to_replicas=129729 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[131158, 66048, 65532, 65326, 117236, 5, 13451, 65532] avg=65536.0000 max=131158 max_rank=0 min=5 min_rank=5 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[131158, 66048, 65532, 65326, 117236, 5, 13451, 65532] avg=65536.0000 max=131158 max_rank=0 min=5 min_rank=5 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] avg=65536.0000 max=66194 max_rank=2 min=64153 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=105815 max_load_reduction_pct=0.615171 moved_to_replicas=259364 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] avg=65536.0000 max=66194 max_rank=2 min=64153 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=105815 max_load_reduction_pct=0.615171 moved_to_replicas=259364 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] avg=65536.0000 max=66194 max_rank=2 min=64153 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=105815 max_load_reduction_pct=0.615171 moved_to_replicas=259364 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] avg=65536.0000 max=66192 max_rank=0 min=64495 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64966 max_load_reduction_pct=0.495326 moved_to_replicas=116010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] avg=65536.0000 max=66192 max_rank=0 min=64495 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64966 max_load_reduction_pct=0.495326 moved_to_replicas=116010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[131158, 66048, 65532, 65326, 117236, 5, 13451, 65532] avg=65536.0000 max=131158 max_rank=0 min=5 min_rank=5 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[4, 131061, 90055, 131007, 6, 172009, 141, 5] avg=65536.0000 max=172009 max_rank=5 min=4 min_rank=0 imbalance_ratio=2.624649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[4, 131061, 90055, 131007, 6, 172009, 141, 5] avg=65536.0000 max=172009 max_rank=5 min=4 min_rank=0 imbalance_ratio=2.624649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[131158, 66048, 65532, 65326, 117236, 5, 13451, 65532] avg=65536.0000 max=131158 max_rank=0 min=5 min_rank=5 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] avg=65536.0000 max=66192 max_rank=0 min=64495 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64966 max_load_reduction_pct=0.495326 moved_to_replicas=116010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[131158, 66048, 65532, 65326, 117236, 5, 13451, 65532] avg=65536.0000 max=131158 max_rank=0 min=5 min_rank=5 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[131158, 66048, 65532, 65326, 117236, 5, 13451, 65532] avg=65536.0000 max=131158 max_rank=0 min=5 min_rank=5 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] avg=65536.0000 max=66194 max_rank=2 min=64153 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=105815 max_load_reduction_pct=0.615171 moved_to_replicas=259364 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65533, 66192, 66194, 66192, 64875, 66193, 64956, 64153] avg=65536.0000 max=66194 max_rank=2 min=64153 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=105815 max_load_reduction_pct=0.615171 moved_to_replicas=259364 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] avg=65536.0000 max=66192 max_rank=0 min=64495 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64966 max_load_reduction_pct=0.495326 moved_to_replicas=116010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] avg=65536.0000 max=66192 max_rank=0 min=64495 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64966 max_load_reduction_pct=0.495326 moved_to_replicas=116010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] avg=65536.0000 max=66192 max_rank=0 min=64495 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64966 max_load_reduction_pct=0.495326 moved_to_replicas=116010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[131158, 66048, 65532, 65326, 117236, 5, 13451, 65532] avg=65536.0000 max=131158 max_rank=0 min=5 min_rank=5 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[131158, 66048, 65532, 65326, 117236, 5, 13451, 65532] avg=65536.0000 max=131158 max_rank=0 min=5 min_rank=5 imbalance_ratio=2.001312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] avg=65536.0000 max=66192 max_rank=0 min=64495 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64966 max_load_reduction_pct=0.495326 moved_to_replicas=116010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[66192, 66048, 65532, 65326, 66192, 64971, 64495, 65532] avg=65536.0000 max=66192 max_rank=0 min=64495 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64966 max_load_reduction_pct=0.495326 moved_to_replicas=116010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=12 total_expected_recv=788828 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189568 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=12 total_expected_recv=783346 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193920 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=12 total_expected_recv=785650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190848 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=12 total_expected_recv=787753 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188928 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=12 total_expected_recv=787557 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190464 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=12 total_expected_recv=787779 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188672 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=12 total_expected_recv=785506 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191104 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=12 total_expected_recv=785037 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192384 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 15:44:41] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 340.4 | throughput per GPU (TFLOP/s/GPU): 531.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.803103E+00 | loss scale: 1.0 | grad norm: 5.575 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=81920 expected_recv=60945 expected_overflow=0 exact_recv_per_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=81920 expected_recv=60945 actual_valid_recv=62464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=81920 expected_recv=62255 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=81920 expected_recv=62255 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=81920 expected_recv=62255 actual_valid_recv=62592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65667 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65667 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65131 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65117 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65131 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=81920 expected_recv=64751 expected_overflow=0 exact_recv_per_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65117 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65651 expected_overflow=0 exact_recv_per_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=81920 expected_recv=64751 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=81920 expected_recv=64868 expected_overflow=0 exact_recv_per_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65582 expected_overflow=0 exact_recv_per_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65651 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=81920 expected_recv=64890 expected_overflow=0 exact_recv_per_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=81920 expected_recv=64868 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65582 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65550 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=81920 expected_recv=64890 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65309 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65090 expected_overflow=0 exact_recv_per_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=81920 expected_recv=64759 expected_overflow=0 exact_recv_per_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65090 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=81920 expected_recv=64759 actual_valid_recv=65280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65309 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=81920 expected_recv=63782 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=81920 expected_recv=63782 actual_valid_recv=64000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65430 expected_overflow=0 exact_recv_per_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65430 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65671 expected_overflow=0 exact_recv_per_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65482 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=81920 expected_recv=64118 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65020 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=81920 expected_recv=64118 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65020 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65671 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=81920 expected_recv=64065 expected_overflow=0 exact_recv_per_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65579 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=81920 expected_recv=64065 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65507 expected_overflow=0 exact_recv_per_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65507 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65104 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65527 actual_valid_recv=65920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65003 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=81920 expected_recv=64113 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65104 actual_valid_recv=65664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65003 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65522 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65522 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=81920 expected_recv=64010 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=81920 expected_recv=64113 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=81920 expected_recv=64010 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=81920 expected_recv=64869 expected_overflow=0 exact_recv_per_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=81920 expected_recv=64869 actual_valid_recv=65280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65348 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=81920 expected_recv=64958 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65348 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65065 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=81920 expected_recv=64958 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65065 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65527 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65435 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64906 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65408 expected_overflow=0 exact_recv_per_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65504 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65408 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65565 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65443 expected_overflow=0 exact_recv_per_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65783 expected_overflow=0 exact_recv_per_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65443 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64801 expected_overflow=0 exact_recv_per_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65783 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65557 expected_overflow=0 exact_recv_per_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65571 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64906 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65571 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64862 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64862 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65683 expected_overflow=0 exact_recv_per_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64801 actual_valid_recv=65152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65557 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65683 actual_valid_recv=66304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65442 expected_overflow=0 exact_recv_per_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65442 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65562 expected_overflow=0 exact_recv_per_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=81920 expected_recv=64783 expected_overflow=0 exact_recv_per_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65543 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65562 actual_valid_recv=65664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=81920 expected_recv=64783 actual_valid_recv=65152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=81920 expected_recv=62917 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=81920 expected_recv=62917 actual_valid_recv=63360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=63360 slack_slots=18560 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65934 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=81920 expected_recv=63181 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65506 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65474 expected_overflow=0 exact_recv_per_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65034 expected_overflow=0 exact_recv_per_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65474 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65955 expected_overflow=0 exact_recv_per_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65034 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=81920 expected_recv=62490 expected_overflow=0 exact_recv_per_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65955 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65934 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=81920 expected_recv=63181 actual_valid_recv=63488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65548 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=81920 expected_recv=62490 actual_valid_recv=62848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=62848 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65624 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65552 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65543 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=81920 expected_recv=64254 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=81920 expected_recv=64254 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=81920 expected_recv=64866 expected_overflow=0 exact_recv_per_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=81920 expected_recv=62819 expected_overflow=0 exact_recv_per_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=81920 expected_recv=62819 actual_valid_recv=62848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=81920 expected_recv=62613 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=62848 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66195 expected_overflow=0 exact_recv_per_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65305 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=81920 expected_recv=62613 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=81920 expected_recv=66195 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65305 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64519 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=81920 expected_recv=64866 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64519 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64652 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64811 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65410 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64652 actual_valid_recv=64896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=81920 expected_recv=64811 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65410 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[48793, 119899, 86813, 27057, 145327, 23435, 37031, 35933] avg=65536.0000 max=145327 max_rank=4 min=23435 min_rank=5 imbalance_ratio=2.217514 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[47209, 121406, 88289, 24132, 140451, 23293, 40937, 38571] avg=65536.0000 max=140451 max_rank=4 min=23293 min_rank=5 imbalance_ratio=2.143112 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[60945, 66191, 66192, 66191, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=60945 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=74258 max_load_reduction_pct=0.528711 moved_to_replicas=151570 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[253377, 89251, 2489, 131058, 38317, 12, 20, 9764] avg=65536.0000 max=253377 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.866226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[66191, 66191, 62255, 66192, 66194, 65536, 65538, 66191] avg=65536.0000 max=66194 max_rank=4 min=62255 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=187183 max_load_reduction_pct=0.738753 moved_to_replicas=275110 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[25, 65667, 196420, 65131, 10, 27, 420, 196588] avg=65536.0000 max=196588 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[64890, 65667, 66192, 65131, 65541, 65558, 65117, 66192] avg=65536.0000 max=66192 max_rank=2 min=64890 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=130396 max_load_reduction_pct=0.663296 moved_to_replicas=260624 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[6, 19, 537, 65534, 124727, 63573, 7866, 262026] avg=65536.0000 max=262026 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.998199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65550, 65309, 65534, 66192, 63782, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195834 max_load_reduction_pct=0.747384 moved_to_replicas=254369 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[200469, 65482, 61975, 45, 130916, 6, 1277, 64118] avg=65536.0000 max=200469 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.058914 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65482, 65020, 65558, 66192, 65534, 66192, 64118] avg=65536.0000 max=66192 max_rank=0 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=134277 max_load_reduction_pct=0.669814 moved_to_replicas=199001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65537, 6, 44750, 64113, 129640, 6, 154713, 65523] avg=65536.0000 max=154713 max_rank=6 min=6 min_rank=1 imbalance_ratio=2.360733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65537, 65535, 65003, 64113, 66192, 66192, 66193, 65523] avg=65536.0000 max=66193 max_rank=6 min=64113 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=88520 max_load_reduction_pct=0.572156 moved_to_replicas=151969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[711, 131253, 64958, 131052, 65527, 65435, 4, 65348] avg=65536.0000 max=131253 max_rank=1 min=4 min_rank=6 imbalance_ratio=2.002762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65571, 66192, 64958, 66192, 65527, 65435, 65065, 65348] avg=65536.0000 max=66192 max_rank=1 min=64958 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65061 max_load_reduction_pct=0.495692 moved_to_replicas=129921 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[131049, 24487, 65535, 106610, 65530, 65534, 5, 65538] avg=65536.0000 max=131049 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999649 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[66192, 64906, 65535, 66191, 65530, 65534, 64862, 65538] avg=65536.0000 max=66192 max_rank=0 min=64862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=105275 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65541, 52, 65553, 137773, 61674, 26583, 36428, 130684] avg=65536.0000 max=137773 max_rank=3 min=52 min_rank=1 imbalance_ratio=2.102249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65541, 65509, 65553, 66192, 62917, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=62917 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=71581 max_load_reduction_pct=0.519558 moved_to_replicas=136073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65548, 131440, 65523, 20651, 686, 43915, 65526, 130999] avg=65536.0000 max=131440 max_rank=1 min=686 min_rank=4 imbalance_ratio=2.005615 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65548, 66192, 65523, 66192, 65934, 63181, 65526, 66192] avg=65536.0000 max=66192 max_rank=1 min=63181 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65248 max_load_reduction_pct=0.496409 moved_to_replicas=130055 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[5, 196584, 65524, 65533, 6307, 168560, 21771, 4] avg=65536.0000 max=196584 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[64866, 66192, 65524, 65533, 66192, 66192, 64254, 65535] avg=65536.0000 max=66192 max_rank=1 min=64254 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130392 max_load_reduction_pct=0.663289 moved_to_replicas=232760 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[131758, 74680, 121807, 116, 130849, 11, 548, 64519] avg=65536.0000 max=131758 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.010468 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66191, 64811, 66192, 65539, 64652, 64519] avg=65536.0000 max=66192 max_rank=0 min=64519 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65566 max_load_reduction_pct=0.497624 moved_to_replicas=194326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=12 total_expected_recv=789799 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186496 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=12 total_expected_recv=787732 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189440 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=12 total_expected_recv=784868 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192640 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=12 total_expected_recv=783255 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194048 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=12 total_expected_recv=788795 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188800 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=12 total_expected_recv=787830 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189952 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=12 total_expected_recv=785975 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191616 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=12 total_expected_recv=783202 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193920 peak_demand=66192 required_capacity=1.010010 - [2026-08-26 14:59:21] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 338.8 | throughput per GPU (TFLOP/s/GPU): 533.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.528715E+00 | loss scale: 1.0 | grad norm: 5.450 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=81920 expected_recv=60943 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67840 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67840 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=81920 expected_recv=60943 actual_valid_recv=62464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65986 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=81920 expected_recv=63349 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65115 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65737 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65986 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=81920 expected_recv=63349 actual_valid_recv=64128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65115 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65737 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[48793, 119899, 86813, 27057, 145327, 23435, 37031, 35933] avg=65536.0000 max=145327 max_rank=4 min=23435 min_rank=5 imbalance_ratio=2.217514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[48793, 119899, 86813, 27057, 145327, 23435, 37031, 35933] avg=65536.0000 max=145327 max_rank=4 min=23435 min_rank=5 imbalance_ratio=2.217514 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[48793, 119899, 86813, 27057, 145327, 23435, 37031, 35933] avg=65536.0000 max=145327 max_rank=4 min=23435 min_rank=5 imbalance_ratio=2.217514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[48793, 119899, 86813, 27057, 145327, 23435, 37031, 35933] avg=65536.0000 max=145327 max_rank=4 min=23435 min_rank=5 imbalance_ratio=2.217514 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=79134 max_load_reduction_pct=0.544524 moved_to_replicas=153463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=79134 max_load_reduction_pct=0.544524 moved_to_replicas=153463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=79134 max_load_reduction_pct=0.544524 moved_to_replicas=153463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=79134 max_load_reduction_pct=0.544524 moved_to_replicas=153463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=79134 max_load_reduction_pct=0.544524 moved_to_replicas=153463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[48793, 119899, 86813, 27057, 145327, 23435, 37031, 35933] avg=65536.0000 max=145327 max_rank=4 min=23435 min_rank=5 imbalance_ratio=2.217514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[48793, 119899, 86813, 27057, 145327, 23435, 37031, 35933] avg=65536.0000 max=145327 max_rank=4 min=23435 min_rank=5 imbalance_ratio=2.217514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[48793, 119899, 86813, 27057, 145327, 23435, 37031, 35933] avg=65536.0000 max=145327 max_rank=4 min=23435 min_rank=5 imbalance_ratio=2.217514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[227910, 69624, 376, 131058, 93924, 10, 21, 1365] avg=65536.0000 max=227910 max_rank=0 min=10 min_rank=5 imbalance_ratio=3.477631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[227910, 69624, 376, 131058, 93924, 10, 21, 1365] avg=65536.0000 max=227910 max_rank=0 min=10 min_rank=5 imbalance_ratio=3.477631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=79134 max_load_reduction_pct=0.544524 moved_to_replicas=153463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=79134 max_load_reduction_pct=0.544524 moved_to_replicas=153463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[227910, 69624, 376, 131058, 93924, 10, 21, 1365] avg=65536.0000 max=227910 max_rank=0 min=10 min_rank=5 imbalance_ratio=3.477631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[227910, 69624, 376, 131058, 93924, 10, 21, 1365] avg=65536.0000 max=227910 max_rank=0 min=10 min_rank=5 imbalance_ratio=3.477631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[227910, 69624, 376, 131058, 93924, 10, 21, 1365] avg=65536.0000 max=227910 max_rank=0 min=10 min_rank=5 imbalance_ratio=3.477631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] avg=65536.0000 max=66192 max_rank=0 min=62255 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=161718 max_load_reduction_pct=0.709570 moved_to_replicas=257748 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66192, 66193, 66191, 60944] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=79134 max_load_reduction_pct=0.544524 moved_to_replicas=153463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] avg=65536.0000 max=66192 max_rank=0 min=62255 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=161718 max_load_reduction_pct=0.709570 moved_to_replicas=257748 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] avg=65536.0000 max=66192 max_rank=0 min=62255 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=161718 max_load_reduction_pct=0.709570 moved_to_replicas=257748 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] avg=65536.0000 max=66192 max_rank=0 min=62255 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=161718 max_load_reduction_pct=0.709570 moved_to_replicas=257748 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] avg=65536.0000 max=66192 max_rank=0 min=62255 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=161718 max_load_reduction_pct=0.709570 moved_to_replicas=257748 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[227910, 69624, 376, 131058, 93924, 10, 21, 1365] avg=65536.0000 max=227910 max_rank=0 min=10 min_rank=5 imbalance_ratio=3.477631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[227910, 69624, 376, 131058, 93924, 10, 21, 1365] avg=65536.0000 max=227910 max_rank=0 min=10 min_rank=5 imbalance_ratio=3.477631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[51, 65651, 196405, 65511, 10, 15, 69, 196576] avg=65536.0000 max=196576 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[51, 65651, 196405, 65511, 10, 15, 69, 196576] avg=65536.0000 max=196576 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[227910, 69624, 376, 131058, 93924, 10, 21, 1365] avg=65536.0000 max=227910 max_rank=0 min=10 min_rank=5 imbalance_ratio=3.477631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] avg=65536.0000 max=66192 max_rank=0 min=62255 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=161718 max_load_reduction_pct=0.709570 moved_to_replicas=257748 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] avg=65536.0000 max=66192 max_rank=0 min=62255 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=161718 max_load_reduction_pct=0.709570 moved_to_replicas=257748 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[51, 65651, 196405, 65511, 10, 15, 69, 196576] avg=65536.0000 max=196576 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[51, 65651, 196405, 65511, 10, 15, 69, 196576] avg=65536.0000 max=196576 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[51, 65651, 196405, 65511, 10, 15, 69, 196576] avg=65536.0000 max=196576 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] avg=65536.0000 max=66192 max_rank=2 min=64751 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130384 max_load_reduction_pct=0.663275 moved_to_replicas=260597 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] avg=65536.0000 max=66192 max_rank=2 min=64751 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130384 max_load_reduction_pct=0.663275 moved_to_replicas=260597 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[66192, 66192, 62255, 66192, 66192, 65537, 65536, 66192] avg=65536.0000 max=66192 max_rank=0 min=62255 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=161718 max_load_reduction_pct=0.709570 moved_to_replicas=257748 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] avg=65536.0000 max=66192 max_rank=2 min=64751 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130384 max_load_reduction_pct=0.663275 moved_to_replicas=260597 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] avg=65536.0000 max=66192 max_rank=2 min=64751 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130384 max_load_reduction_pct=0.663275 moved_to_replicas=260597 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] avg=65536.0000 max=66192 max_rank=2 min=64751 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130384 max_load_reduction_pct=0.663275 moved_to_replicas=260597 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[51, 65651, 196405, 65511, 10, 15, 69, 196576] avg=65536.0000 max=196576 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[51, 65651, 196405, 65511, 10, 15, 69, 196576] avg=65536.0000 max=196576 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[6, 24, 219, 65535, 122648, 65430, 8301, 262125] avg=65536.0000 max=262125 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[6, 24, 219, 65535, 122648, 65430, 8301, 262125] avg=65536.0000 max=262125 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[51, 65651, 196405, 65511, 10, 15, 69, 196576] avg=65536.0000 max=196576 max_rank=7 min=10 min_rank=4 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[6, 24, 219, 65535, 122648, 65430, 8301, 262125] avg=65536.0000 max=262125 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] avg=65536.0000 max=66192 max_rank=2 min=64751 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130384 max_load_reduction_pct=0.663275 moved_to_replicas=260597 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] avg=65536.0000 max=66192 max_rank=2 min=64751 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130384 max_load_reduction_pct=0.663275 moved_to_replicas=260597 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[6, 24, 219, 65535, 122648, 65430, 8301, 262125] avg=65536.0000 max=262125 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[6, 24, 219, 65535, 122648, 65430, 8301, 262125] avg=65536.0000 max=262125 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] avg=65536.0000 max=66192 max_rank=7 min=64759 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=195933 max_load_reduction_pct=0.747479 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] avg=65536.0000 max=66192 max_rank=7 min=64759 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=195933 max_load_reduction_pct=0.747479 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[65582, 65651, 66192, 65511, 65541, 64868, 64751, 66192] avg=65536.0000 max=66192 max_rank=2 min=64751 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130384 max_load_reduction_pct=0.663275 moved_to_replicas=260597 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] avg=65536.0000 max=66192 max_rank=7 min=64759 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=195933 max_load_reduction_pct=0.747479 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] avg=65536.0000 max=66192 max_rank=7 min=64759 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=195933 max_load_reduction_pct=0.747479 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] avg=65536.0000 max=66192 max_rank=7 min=64759 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=195933 max_load_reduction_pct=0.747479 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[6, 24, 219, 65535, 122648, 65430, 8301, 262125] avg=65536.0000 max=262125 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[6, 24, 219, 65535, 122648, 65430, 8301, 262125] avg=65536.0000 max=262125 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[198010, 65507, 65671, 34, 130817, 6, 178, 64065] avg=65536.0000 max=198010 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.021393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[198010, 65507, 65671, 34, 130817, 6, 178, 64065] avg=65536.0000 max=198010 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.021393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[6, 24, 219, 65535, 122648, 65430, 8301, 262125] avg=65536.0000 max=262125 max_rank=7 min=6 min_rank=0 imbalance_ratio=3.999710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[198010, 65507, 65671, 34, 130817, 6, 178, 64065] avg=65536.0000 max=198010 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.021393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] avg=65536.0000 max=66192 max_rank=7 min=64759 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=195933 max_load_reduction_pct=0.747479 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] avg=65536.0000 max=66192 max_rank=7 min=64759 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=195933 max_load_reduction_pct=0.747479 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[198010, 65507, 65671, 34, 130817, 6, 178, 64065] avg=65536.0000 max=198010 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.021393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[198010, 65507, 65671, 34, 130817, 6, 178, 64065] avg=65536.0000 max=198010 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.021393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] avg=65536.0000 max=66192 max_rank=0 min=64065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131818 max_load_reduction_pct=0.665714 moved_to_replicas=196443 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] avg=65536.0000 max=66192 max_rank=0 min=64065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131818 max_load_reduction_pct=0.665714 moved_to_replicas=196443 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65537, 65555, 65090, 65535, 66190, 65430, 64759, 66192] avg=65536.0000 max=66192 max_rank=7 min=64759 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=195933 max_load_reduction_pct=0.747479 moved_to_replicas=252389 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] avg=65536.0000 max=66192 max_rank=0 min=64065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131818 max_load_reduction_pct=0.665714 moved_to_replicas=196443 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] avg=65536.0000 max=66192 max_rank=0 min=64065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131818 max_load_reduction_pct=0.665714 moved_to_replicas=196443 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] avg=65536.0000 max=66192 max_rank=0 min=64065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131818 max_load_reduction_pct=0.665714 moved_to_replicas=196443 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[198010, 65507, 65671, 34, 130817, 6, 178, 64065] avg=65536.0000 max=198010 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.021393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[198010, 65507, 65671, 34, 130817, 6, 178, 64065] avg=65536.0000 max=198010 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.021393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65542, 6, 62651, 64010, 127470, 4, 139083, 65522] avg=65536.0000 max=139083 max_rank=6 min=4 min_rank=5 imbalance_ratio=2.122238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65542, 6, 62651, 64010, 127470, 4, 139083, 65522] avg=65536.0000 max=139083 max_rank=6 min=4 min_rank=5 imbalance_ratio=2.122238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[198010, 65507, 65671, 34, 130817, 6, 178, 64065] avg=65536.0000 max=198010 max_rank=0 min=6 min_rank=5 imbalance_ratio=3.021393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65542, 6, 62651, 64010, 127470, 4, 139083, 65522] avg=65536.0000 max=139083 max_rank=6 min=4 min_rank=5 imbalance_ratio=2.122238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] avg=65536.0000 max=66192 max_rank=0 min=64065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131818 max_load_reduction_pct=0.665714 moved_to_replicas=196443 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] avg=65536.0000 max=66192 max_rank=0 min=64065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131818 max_load_reduction_pct=0.665714 moved_to_replicas=196443 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] avg=65536.0000 max=66192 max_rank=1 min=64010 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=72891 max_load_reduction_pct=0.524083 moved_to_replicas=134169 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65542, 6, 62651, 64010, 127470, 4, 139083, 65522] avg=65536.0000 max=139083 max_rank=6 min=4 min_rank=5 imbalance_ratio=2.122238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65542, 6, 62651, 64010, 127470, 4, 139083, 65522] avg=65536.0000 max=139083 max_rank=6 min=4 min_rank=5 imbalance_ratio=2.122238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] avg=65536.0000 max=66192 max_rank=1 min=64010 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=72891 max_load_reduction_pct=0.524083 moved_to_replicas=134169 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[66192, 65507, 65671, 65555, 66192, 65527, 65579, 64065] avg=65536.0000 max=66192 max_rank=0 min=64065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131818 max_load_reduction_pct=0.665714 moved_to_replicas=196443 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] avg=65536.0000 max=66192 max_rank=1 min=64010 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=72891 max_load_reduction_pct=0.524083 moved_to_replicas=134169 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65542, 6, 62651, 64010, 127470, 4, 139083, 65522] avg=65536.0000 max=139083 max_rank=6 min=4 min_rank=5 imbalance_ratio=2.122238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] avg=65536.0000 max=66192 max_rank=1 min=64010 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=72891 max_load_reduction_pct=0.524083 moved_to_replicas=134169 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] avg=65536.0000 max=66192 max_rank=1 min=64010 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=72891 max_load_reduction_pct=0.524083 moved_to_replicas=134169 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65542, 6, 62651, 64010, 127470, 4, 139083, 65522] avg=65536.0000 max=139083 max_rank=6 min=4 min_rank=5 imbalance_ratio=2.122238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[577, 131398, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=131398 max_rank=1 min=577 min_rank=0 imbalance_ratio=2.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[577, 131398, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=131398 max_rank=1 min=577 min_rank=0 imbalance_ratio=2.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65542, 6, 62651, 64010, 127470, 4, 139083, 65522] avg=65536.0000 max=139083 max_rank=6 min=4 min_rank=5 imbalance_ratio=2.122238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[577, 131398, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=131398 max_rank=1 min=577 min_rank=0 imbalance_ratio=2.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] avg=65536.0000 max=66192 max_rank=1 min=64010 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=72891 max_load_reduction_pct=0.524083 moved_to_replicas=134169 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] avg=65536.0000 max=66192 max_rank=1 min=64010 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=72891 max_load_reduction_pct=0.524083 moved_to_replicas=134169 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=66192 max_rank=1 min=64869 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.496248 moved_to_replicas=65206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=66192 max_rank=1 min=64869 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.496248 moved_to_replicas=65206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[577, 131398, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=131398 max_rank=1 min=577 min_rank=0 imbalance_ratio=2.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[577, 131398, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=131398 max_rank=1 min=577 min_rank=0 imbalance_ratio=2.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65542, 66192, 65104, 64010, 66192, 65534, 66192, 65522] avg=65536.0000 max=66192 max_rank=1 min=64010 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=72891 max_load_reduction_pct=0.524083 moved_to_replicas=134169 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=66192 max_rank=1 min=64869 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.496248 moved_to_replicas=65206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=66192 max_rank=1 min=64869 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.496248 moved_to_replicas=65206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=66192 max_rank=1 min=64869 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.496248 moved_to_replicas=65206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[577, 131398, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=131398 max_rank=1 min=577 min_rank=0 imbalance_ratio=2.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[577, 131398, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=131398 max_rank=1 min=577 min_rank=0 imbalance_ratio=2.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[130987, 65442, 65537, 65683, 65533, 65543, 6, 65557] avg=65536.0000 max=130987 max_rank=0 min=6 min_rank=6 imbalance_ratio=1.998703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[130987, 65442, 65537, 65683, 65533, 65543, 6, 65557] avg=65536.0000 max=130987 max_rank=0 min=6 min_rank=6 imbalance_ratio=1.998703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[577, 131398, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=131398 max_rank=1 min=577 min_rank=0 imbalance_ratio=2.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[130987, 65442, 65537, 65683, 65533, 65543, 6, 65557] avg=65536.0000 max=130987 max_rank=0 min=6 min_rank=6 imbalance_ratio=1.998703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=66192 max_rank=1 min=64869 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.496248 moved_to_replicas=65206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=66192 max_rank=1 min=64869 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.496248 moved_to_replicas=65206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] avg=65536.0000 max=66192 max_rank=0 min=64801 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64795 max_load_reduction_pct=0.494667 moved_to_replicas=64795 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] avg=65536.0000 max=66192 max_rank=0 min=64801 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64795 max_load_reduction_pct=0.494667 moved_to_replicas=64795 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[130987, 65442, 65537, 65683, 65533, 65543, 6, 65557] avg=65536.0000 max=130987 max_rank=0 min=6 min_rank=6 imbalance_ratio=1.998703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[130987, 65442, 65537, 65683, 65533, 65543, 6, 65557] avg=65536.0000 max=130987 max_rank=0 min=6 min_rank=6 imbalance_ratio=1.998703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65783, 66192, 64869, 65565, 65524, 65443, 65504, 65408] avg=65536.0000 max=66192 max_rank=1 min=64869 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.496248 moved_to_replicas=65206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] avg=65536.0000 max=66192 max_rank=0 min=64801 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64795 max_load_reduction_pct=0.494667 moved_to_replicas=64795 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] avg=65536.0000 max=66192 max_rank=0 min=64801 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64795 max_load_reduction_pct=0.494667 moved_to_replicas=64795 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] avg=65536.0000 max=66192 max_rank=0 min=64801 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64795 max_load_reduction_pct=0.494667 moved_to_replicas=64795 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[130987, 65442, 65537, 65683, 65533, 65543, 6, 65557] avg=65536.0000 max=130987 max_rank=0 min=6 min_rank=6 imbalance_ratio=1.998703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[130987, 65442, 65537, 65683, 65533, 65543, 6, 65557] avg=65536.0000 max=130987 max_rank=0 min=6 min_rank=6 imbalance_ratio=1.998703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65544, 25, 65562, 171278, 65474, 369, 64783, 91253] avg=65536.0000 max=171278 max_rank=3 min=25 min_rank=1 imbalance_ratio=2.613495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65544, 25, 65562, 171278, 65474, 369, 64783, 91253] avg=65536.0000 max=171278 max_rank=3 min=25 min_rank=1 imbalance_ratio=2.613495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[130987, 65442, 65537, 65683, 65533, 65543, 6, 65557] avg=65536.0000 max=130987 max_rank=0 min=6 min_rank=6 imbalance_ratio=1.998703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65544, 25, 65562, 171278, 65474, 369, 64783, 91253] avg=65536.0000 max=171278 max_rank=3 min=25 min_rank=1 imbalance_ratio=2.613495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] avg=65536.0000 max=66192 max_rank=0 min=64801 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64795 max_load_reduction_pct=0.494667 moved_to_replicas=64795 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] avg=65536.0000 max=66192 max_rank=0 min=64801 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64795 max_load_reduction_pct=0.494667 moved_to_replicas=64795 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] avg=65536.0000 max=66193 max_rank=3 min=64783 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=105085 max_load_reduction_pct=0.613535 moved_to_replicas=130147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] avg=65536.0000 max=66193 max_rank=3 min=64783 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=105085 max_load_reduction_pct=0.613535 moved_to_replicas=130147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65544, 25, 65562, 171278, 65474, 369, 64783, 91253] avg=65536.0000 max=171278 max_rank=3 min=25 min_rank=1 imbalance_ratio=2.613495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65544, 25, 65562, 171278, 65474, 369, 64783, 91253] avg=65536.0000 max=171278 max_rank=3 min=25 min_rank=1 imbalance_ratio=2.613495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[66192, 65442, 65537, 65683, 65533, 65543, 64801, 65557] avg=65536.0000 max=66192 max_rank=0 min=64801 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64795 max_load_reduction_pct=0.494667 moved_to_replicas=64795 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] avg=65536.0000 max=66193 max_rank=3 min=64783 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=105085 max_load_reduction_pct=0.613535 moved_to_replicas=130147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] avg=65536.0000 max=66193 max_rank=3 min=64783 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=105085 max_load_reduction_pct=0.613535 moved_to_replicas=130147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] avg=65536.0000 max=66193 max_rank=3 min=64783 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=105085 max_load_reduction_pct=0.613535 moved_to_replicas=130147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65544, 25, 65562, 171278, 65474, 369, 64783, 91253] avg=65536.0000 max=171278 max_rank=3 min=25 min_rank=1 imbalance_ratio=2.613495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65544, 25, 65562, 171278, 65474, 369, 64783, 91253] avg=65536.0000 max=171278 max_rank=3 min=25 min_rank=1 imbalance_ratio=2.613495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65552, 132587, 65523, 56080, 17657, 55438, 443, 131008] avg=65536.0000 max=132587 max_rank=1 min=443 min_rank=6 imbalance_ratio=2.023117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65552, 132587, 65523, 56080, 17657, 55438, 443, 131008] avg=65536.0000 max=132587 max_rank=1 min=443 min_rank=6 imbalance_ratio=2.023117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65544, 25, 65562, 171278, 65474, 369, 64783, 91253] avg=65536.0000 max=171278 max_rank=3 min=25 min_rank=1 imbalance_ratio=2.613495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] avg=65536.0000 max=66193 max_rank=3 min=64783 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=105085 max_load_reduction_pct=0.613535 moved_to_replicas=130147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65552, 132587, 65523, 56080, 17657, 55438, 443, 131008] avg=65536.0000 max=132587 max_rank=1 min=443 min_rank=6 imbalance_ratio=2.023117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] avg=65536.0000 max=66193 max_rank=3 min=64783 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=105085 max_load_reduction_pct=0.613535 moved_to_replicas=130147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] avg=65536.0000 max=66193 max_rank=5 min=62490 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=66394 max_load_reduction_pct=0.500758 moved_to_replicas=131211 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] avg=65536.0000 max=66193 max_rank=5 min=62490 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=66394 max_load_reduction_pct=0.500758 moved_to_replicas=131211 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65552, 132587, 65523, 56080, 17657, 55438, 443, 131008] avg=65536.0000 max=132587 max_rank=1 min=443 min_rank=6 imbalance_ratio=2.023117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65552, 132587, 65523, 56080, 17657, 55438, 443, 131008] avg=65536.0000 max=132587 max_rank=1 min=443 min_rank=6 imbalance_ratio=2.023117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65544, 65506, 65562, 66193, 65474, 65034, 64783, 66192] avg=65536.0000 max=66193 max_rank=3 min=64783 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=105085 max_load_reduction_pct=0.613535 moved_to_replicas=130147 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] avg=65536.0000 max=66193 max_rank=5 min=62490 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=66394 max_load_reduction_pct=0.500758 moved_to_replicas=131211 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] avg=65536.0000 max=66193 max_rank=5 min=62490 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=66394 max_load_reduction_pct=0.500758 moved_to_replicas=131211 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] avg=65536.0000 max=66193 max_rank=5 min=62490 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=66394 max_load_reduction_pct=0.500758 moved_to_replicas=131211 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65552, 132587, 65523, 56080, 17657, 55438, 443, 131008] avg=65536.0000 max=132587 max_rank=1 min=443 min_rank=6 imbalance_ratio=2.023117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65552, 132587, 65523, 56080, 17657, 55438, 443, 131008] avg=65536.0000 max=132587 max_rank=1 min=443 min_rank=6 imbalance_ratio=2.023117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[5, 131087, 72697, 65543, 50163, 145670, 59119, 4] avg=65536.0000 max=145670 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.222748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[5, 131087, 72697, 65543, 50163, 145670, 59119, 4] avg=65536.0000 max=145670 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.222748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65552, 132587, 65523, 56080, 17657, 55438, 443, 131008] avg=65536.0000 max=132587 max_rank=1 min=443 min_rank=6 imbalance_ratio=2.023117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] avg=65536.0000 max=66193 max_rank=5 min=62490 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=66394 max_load_reduction_pct=0.500758 moved_to_replicas=131211 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[5, 131087, 72697, 65543, 50163, 145670, 59119, 4] avg=65536.0000 max=145670 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.222748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] avg=65536.0000 max=66193 max_rank=5 min=62490 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=66394 max_load_reduction_pct=0.500758 moved_to_replicas=131211 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] avg=65536.0000 max=66195 max_rank=0 min=62819 min_rank=4 imbalance_ratio=1.010056 max_load_reduction=79475 max_load_reduction_pct=0.545582 moved_to_replicas=150878 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] avg=65536.0000 max=66195 max_rank=0 min=62819 min_rank=4 imbalance_ratio=1.010056 max_load_reduction=79475 max_load_reduction_pct=0.545582 moved_to_replicas=150878 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[5, 131087, 72697, 65543, 50163, 145670, 59119, 4] avg=65536.0000 max=145670 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.222748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[5, 131087, 72697, 65543, 50163, 145670, 59119, 4] avg=65536.0000 max=145670 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.222748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65552, 66192, 65523, 62490, 66191, 66193, 65955, 66192] avg=65536.0000 max=66193 max_rank=5 min=62490 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=66394 max_load_reduction_pct=0.500758 moved_to_replicas=131211 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] avg=65536.0000 max=66195 max_rank=0 min=62819 min_rank=4 imbalance_ratio=1.010056 max_load_reduction=79475 max_load_reduction_pct=0.545582 moved_to_replicas=150878 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] avg=65536.0000 max=66195 max_rank=0 min=62819 min_rank=4 imbalance_ratio=1.010056 max_load_reduction=79475 max_load_reduction_pct=0.545582 moved_to_replicas=150878 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] avg=65536.0000 max=66195 max_rank=0 min=62819 min_rank=4 imbalance_ratio=1.010056 max_load_reduction=79475 max_load_reduction_pct=0.545582 moved_to_replicas=150878 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[5, 131087, 72697, 65543, 50163, 145670, 59119, 4] avg=65536.0000 max=145670 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.222748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[5, 131087, 72697, 65543, 50163, 145670, 59119, 4] avg=65536.0000 max=145670 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.222748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[131591, 84046, 114804, 66, 128412, 11, 53, 65305] avg=65536.0000 max=131591 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[131591, 84046, 114804, 66, 128412, 11, 53, 65305] avg=65536.0000 max=131591 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[5, 131087, 72697, 65543, 50163, 145670, 59119, 4] avg=65536.0000 max=145670 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.222748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[131591, 84046, 114804, 66, 128412, 11, 53, 65305] avg=65536.0000 max=131591 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] avg=65536.0000 max=66195 max_rank=0 min=62819 min_rank=4 imbalance_ratio=1.010056 max_load_reduction=79475 max_load_reduction_pct=0.545582 moved_to_replicas=150878 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] avg=65536.0000 max=66195 max_rank=0 min=62819 min_rank=4 imbalance_ratio=1.010056 max_load_reduction=79475 max_load_reduction_pct=0.545582 moved_to_replicas=150878 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] avg=65536.0000 max=66193 max_rank=2 min=62613 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65398 max_load_reduction_pct=0.496979 moved_to_replicas=194085 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] avg=65536.0000 max=66193 max_rank=2 min=62613 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65398 max_load_reduction_pct=0.496979 moved_to_replicas=194085 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[131591, 84046, 114804, 66, 128412, 11, 53, 65305] avg=65536.0000 max=131591 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[131591, 84046, 114804, 66, 128412, 11, 53, 65305] avg=65536.0000 max=131591 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[66195, 66192, 66192, 65543, 62819, 66191, 65624, 65532] avg=65536.0000 max=66195 max_rank=0 min=62819 min_rank=4 imbalance_ratio=1.010056 max_load_reduction=79475 max_load_reduction_pct=0.545582 moved_to_replicas=150878 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] avg=65536.0000 max=66193 max_rank=2 min=62613 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65398 max_load_reduction_pct=0.496979 moved_to_replicas=194085 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] avg=65536.0000 max=66193 max_rank=2 min=62613 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65398 max_load_reduction_pct=0.496979 moved_to_replicas=194085 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[131591, 84046, 114804, 66, 128412, 11, 53, 65305] avg=65536.0000 max=131591 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[131591, 84046, 114804, 66, 128412, 11, 53, 65305] avg=65536.0000 max=131591 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] avg=65536.0000 max=66193 max_rank=2 min=62613 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65398 max_load_reduction_pct=0.496979 moved_to_replicas=194085 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[131591, 84046, 114804, 66, 128412, 11, 53, 65305] avg=65536.0000 max=131591 max_rank=0 min=11 min_rank=5 imbalance_ratio=2.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] avg=65536.0000 max=66193 max_rank=2 min=62613 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65398 max_load_reduction_pct=0.496979 moved_to_replicas=194085 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] avg=65536.0000 max=66193 max_rank=2 min=62613 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65398 max_load_reduction_pct=0.496979 moved_to_replicas=194085 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[66192, 66192, 66193, 66191, 66192, 65410, 62613, 65305] avg=65536.0000 max=66193 max_rank=2 min=62613 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65398 max_load_reduction_pct=0.496979 moved_to_replicas=194085 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=12 total_expected_recv=784380 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193024 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=12 total_expected_recv=782288 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194816 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=12 total_expected_recv=783293 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193792 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=12 total_expected_recv=784659 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193024 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=12 total_expected_recv=791006 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=12 total_expected_recv=786903 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190848 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=12 total_expected_recv=788232 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190208 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=12 total_expected_recv=790695 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186240 peak_demand=66195 required_capacity=1.010056 + [2026-08-26 15:44:42] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 339.2 | throughput per GPU (TFLOP/s/GPU): 533.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.537196E+00 | loss scale: 1.0 | grad norm: 5.401 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=81920 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=81920 expected_recv=60944 actual_valid_recv=62720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62720 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67840 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=81920 expected_recv=64923 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65458 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=81920 expected_recv=64923 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=81920 expected_recv=63618 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65458 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=81920 expected_recv=63618 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64968 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65041 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64540 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65622 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64968 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65041 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65384 expected_overflow=0 exact_recv_per_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65603 expected_overflow=0 exact_recv_per_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65384 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64327 expected_overflow=0 exact_recv_per_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64849 expected_overflow=0 exact_recv_per_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65603 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64327 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64849 actual_valid_recv=65280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=81920 expected_recv=64540 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=81920 expected_recv=64006 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65025 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65575 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65574 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=81920 expected_recv=64006 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65025 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=81920 expected_recv=64057 expected_overflow=0 exact_recv_per_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=81920 expected_recv=64057 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65453 expected_overflow=0 exact_recv_per_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65571 expected_overflow=0 exact_recv_per_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65453 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65714 expected_overflow=0 exact_recv_per_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65571 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65714 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65577 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65473 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66156 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65473 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65620 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66156 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=81920 expected_recv=63396 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65726 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=81920 expected_recv=63396 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65726 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65087 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=81920 expected_recv=63982 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65087 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=81920 expected_recv=63982 actual_valid_recv=64512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=81920 expected_recv=64834 expected_overflow=0 exact_recv_per_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65505 expected_overflow=0 exact_recv_per_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=81920 expected_recv=64264 expected_overflow=0 exact_recv_per_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=81920 expected_recv=64834 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65505 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=81920 expected_recv=64264 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=81920 expected_recv=62892 expected_overflow=0 exact_recv_per_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65575 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=81920 expected_recv=62892 actual_valid_recv=63360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=63360 slack_slots=18560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65580 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65567 actual_valid_recv=66304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65640 actual_valid_recv=66304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65393 expected_overflow=0 exact_recv_per_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65079 expected_overflow=0 exact_recv_per_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65393 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65508 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65079 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65218 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65218 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66078 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=81920 expected_recv=63806 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=81920 expected_recv=66078 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65743 expected_overflow=0 exact_recv_per_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=81920 expected_recv=63806 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65743 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65037 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65037 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65694 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=81920 expected_recv=64592 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65694 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65423 expected_overflow=0 exact_recv_per_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65517 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65423 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=81920 expected_recv=64592 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65036 expected_overflow=0 exact_recv_per_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65706 expected_overflow=0 exact_recv_per_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=81920 expected_recv=64565 expected_overflow=0 exact_recv_per_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65036 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65706 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65742 expected_overflow=0 exact_recv_per_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=81920 expected_recv=64565 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=81920 expected_recv=63718 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65742 actual_valid_recv=65920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=81920 expected_recv=64707 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=81920 expected_recv=63718 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=81920 expected_recv=64707 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=81920 expected_recv=62906 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=81920 expected_recv=62906 actual_valid_recv=63616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65516 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65568 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=81920 expected_recv=64087 expected_overflow=0 exact_recv_per_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=81920 expected_recv=64087 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65430 expected_overflow=0 exact_recv_per_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65517 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65430 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=81920 expected_recv=64834 expected_overflow=0 exact_recv_per_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=81920 expected_recv=64834 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65057 expected_overflow=0 exact_recv_per_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65057 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65360 expected_overflow=0 exact_recv_per_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65593 expected_overflow=0 exact_recv_per_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=81920 expected_recv=63618 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=81920 expected_recv=63618 actual_valid_recv=64000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=81920 expected_recv=64844 expected_overflow=0 exact_recv_per_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65360 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65593 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=81920 expected_recv=64844 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=81920 expected_recv=64626 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=81920 expected_recv=64626 actual_valid_recv=64896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65647 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=81920 expected_recv=64079 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65647 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=81920 expected_recv=64079 actual_valid_recv=64640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65168 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65168 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=81920 expected_recv=61819 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65974 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=81920 expected_recv=62502 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=81920 expected_recv=61819 actual_valid_recv=62208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65974 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=81920 expected_recv=62502 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=62208 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65216 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65216 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[42855, 122200, 83905, 24093, 152225, 26135, 35236, 37639] avg=65536.0000 max=152225 max_rank=4 min=24093 min_rank=3 imbalance_ratio=2.322769 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65610 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[42855, 122200, 83905, 24093, 152225, 26135, 35236, 37639] avg=65536.0000 max=152225 max_rank=4 min=24093 min_rank=3 imbalance_ratio=2.322769 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[41336, 123332, 86315, 21605, 146427, 26096, 38675, 40502] avg=65536.0000 max=146427 max_rank=4 min=21605 min_rank=3 imbalance_ratio=2.234299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[66192, 66192, 66192, 66193, 66192, 60943, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=60943 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=80234 max_load_reduction_pct=0.547945 moved_to_replicas=157498 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[257284, 65986, 4105, 131061, 403, 15, 319, 65115] avg=65536.0000 max=257284 max_rank=0 min=15 min_rank=5 imbalance_ratio=3.925842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[66192, 65986, 66192, 66192, 63349, 65525, 65737, 65115] avg=65536.0000 max=66192 max_rank=0 min=63349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=191092 max_load_reduction_pct=0.742728 moved_to_replicas=255961 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[15, 65622, 179951, 1526, 24, 15949, 64540, 196661] avg=65536.0000 max=196661 max_rank=7 min=15 min_rank=0 imbalance_ratio=3.000809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[65540, 65622, 66193, 66192, 64968, 65041, 64540, 66192] avg=65536.0000 max=66193 max_rank=2 min=64540 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=130468 max_load_reduction_pct=0.663416 moved_to_replicas=244228 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[9, 55, 58508, 65574, 69071, 64006, 5458, 261607] avg=65536.0000 max=261607 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.991806 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65575, 65025, 65574, 66192, 64006, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=64006 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=195415 max_load_reduction_pct=0.746979 moved_to_replicas=198294 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[263045, 65473, 658, 110, 130815, 14, 777, 63396] avg=65536.0000 max=263045 max_rank=0 min=14 min_rank=5 imbalance_ratio=4.013748 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65473, 66156, 65620, 66192, 65533, 65726, 63396] avg=65536.0000 max=66192 max_rank=0 min=63396 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=196853 max_load_reduction_pct=0.748362 moved_to_replicas=261476 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65580, 17, 30276, 63982, 130574, 12, 168316, 65531] avg=65536.0000 max=168316 max_rank=6 min=12 min_rank=5 imbalance_ratio=2.568298 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65580, 65087, 66191, 63982, 66192, 65534, 66191, 65531] avg=65536.0000 max=66192 max_rank=4 min=63982 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=102124 max_load_reduction_pct=0.606740 moved_to_replicas=166506 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65743, 132830, 135, 131019, 65521, 65218, 16, 63806] avg=65536.0000 max=132830 max_rank=1 min=16 min_rank=6 imbalance_ratio=2.026825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65743, 66192, 66078, 66192, 65521, 65218, 65538, 63806] avg=65536.0000 max=66192 max_rank=1 min=63806 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66638 max_load_reduction_pct=0.501679 moved_to_replicas=131465 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[130773, 643, 65531, 130586, 65539, 65511, 11, 65694] avg=65536.0000 max=130773 max_rank=0 min=11 min_rank=6 imbalance_ratio=1.995438 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[66192, 65037, 65531, 66192, 65539, 65511, 64592, 65694] avg=65536.0000 max=66192 max_rank=0 min=64592 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64581 max_load_reduction_pct=0.493840 moved_to_replicas=128975 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65542, 76, 65553, 129834, 61135, 62702, 33469, 105977] avg=65536.0000 max=129834 max_rank=3 min=76 min_rank=1 imbalance_ratio=1.981110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65542, 63718, 65553, 66192, 66192, 64707, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=63718 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=63641 max_load_reduction_pct=0.490172 moved_to_replicas=103427 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65544, 164510, 65516, 41, 2479, 131, 95125, 130942] avg=65536.0000 max=164510 max_rank=1 min=41 min_rank=3 imbalance_ratio=2.510223 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65544, 66191, 65516, 65555, 66192, 62906, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=62906 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98318 max_load_reduction_pct=0.597641 moved_to_replicas=192001 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[13, 196545, 65524, 55357, 59551, 131061, 16225, 12] avg=65536.0000 max=196545 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.999039 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[64844, 66192, 65524, 66192, 63618, 66192, 66192, 65534] avg=65536.0000 max=66192 max_rank=1 min=63618 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130353 max_load_reduction_pct=0.663222 moved_to_replicas=195222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[137169, 79129, 61338, 128, 124490, 64626, 55996, 1412] avg=65536.0000 max=137169 max_rank=0 min=128 min_rank=3 imbalance_ratio=2.093033 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66193, 64079, 65647, 66192, 64626, 66191, 65168] avg=65536.0000 max=66193 max_rank=1 min=64079 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=70976 max_load_reduction_pct=0.517435 moved_to_replicas=142212 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=12 total_expected_recv=775742 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=198912 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=12 total_expected_recv=787458 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186880 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=12 total_expected_recv=789723 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=12 total_expected_recv=786339 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188928 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=12 total_expected_recv=789474 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185088 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=12 total_expected_recv=785205 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189568 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=12 total_expected_recv=788230 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188288 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=12 total_expected_recv=789285 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185344 peak_demand=66192 required_capacity=1.010010 - [2026-08-26 14:59:21] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 340.3 | throughput per GPU (TFLOP/s/GPU): 531.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.429726E+00 | loss scale: 1.0 | grad norm: 5.282 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=81920 expected_recv=63722 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=81920 expected_recv=63722 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=81920 expected_recv=63413 expected_overflow=0 exact_recv_per_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65610 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=86032 max_load_reduction_pct=0.565163 moved_to_replicas=159754 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=86032 max_load_reduction_pct=0.565163 moved_to_replicas=159754 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[42855, 122200, 83905, 24093, 152225, 26135, 35236, 37639] avg=65536.0000 max=152225 max_rank=4 min=24093 min_rank=3 imbalance_ratio=2.322769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[42855, 122200, 83905, 24093, 152225, 26135, 35236, 37639] avg=65536.0000 max=152225 max_rank=4 min=24093 min_rank=3 imbalance_ratio=2.322769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[42855, 122200, 83905, 24093, 152225, 26135, 35236, 37639] avg=65536.0000 max=152225 max_rank=4 min=24093 min_rank=3 imbalance_ratio=2.322769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[42855, 122200, 83905, 24093, 152225, 26135, 35236, 37639] avg=65536.0000 max=152225 max_rank=4 min=24093 min_rank=3 imbalance_ratio=2.322769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[190797, 83285, 52, 131063, 55412, 12, 49, 63618] avg=65536.0000 max=190797 max_rank=0 min=12 min_rank=5 imbalance_ratio=2.911331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[42855, 122200, 83905, 24093, 152225, 26135, 35236, 37639] avg=65536.0000 max=152225 max_rank=4 min=24093 min_rank=3 imbalance_ratio=2.322769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[42855, 122200, 83905, 24093, 152225, 26135, 35236, 37639] avg=65536.0000 max=152225 max_rank=4 min=24093 min_rank=3 imbalance_ratio=2.322769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=86032 max_load_reduction_pct=0.565163 moved_to_replicas=159754 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=86032 max_load_reduction_pct=0.565163 moved_to_replicas=159754 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[190797, 83285, 52, 131063, 55412, 12, 49, 63618] avg=65536.0000 max=190797 max_rank=0 min=12 min_rank=5 imbalance_ratio=2.911331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=86032 max_load_reduction_pct=0.565163 moved_to_replicas=159754 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=86032 max_load_reduction_pct=0.565163 moved_to_replicas=159754 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] avg=65536.0000 max=66192 max_rank=0 min=63618 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=124605 max_load_reduction_pct=0.653076 moved_to_replicas=206569 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=86032 max_load_reduction_pct=0.565163 moved_to_replicas=159754 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[66191, 66193, 66191, 66193, 66192, 66192, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=86032 max_load_reduction_pct=0.565163 moved_to_replicas=159754 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] avg=65536.0000 max=66192 max_rank=0 min=63618 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=124605 max_load_reduction_pct=0.653076 moved_to_replicas=206569 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[190797, 83285, 52, 131063, 55412, 12, 49, 63618] avg=65536.0000 max=190797 max_rank=0 min=12 min_rank=5 imbalance_ratio=2.911331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[190797, 83285, 52, 131063, 55412, 12, 49, 63618] avg=65536.0000 max=190797 max_rank=0 min=12 min_rank=5 imbalance_ratio=2.911331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[190797, 83285, 52, 131063, 55412, 12, 49, 63618] avg=65536.0000 max=190797 max_rank=0 min=12 min_rank=5 imbalance_ratio=2.911331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[190797, 83285, 52, 131063, 55412, 12, 49, 63618] avg=65536.0000 max=190797 max_rank=0 min=12 min_rank=5 imbalance_ratio=2.911331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[33, 65603, 128386, 64327, 24, 6714, 62668, 196533] avg=65536.0000 max=196533 max_rank=7 min=24 min_rank=4 imbalance_ratio=2.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] avg=65536.0000 max=66192 max_rank=0 min=63618 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=124605 max_load_reduction_pct=0.653076 moved_to_replicas=206569 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[190797, 83285, 52, 131063, 55412, 12, 49, 63618] avg=65536.0000 max=190797 max_rank=0 min=12 min_rank=5 imbalance_ratio=2.911331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[190797, 83285, 52, 131063, 55412, 12, 49, 63618] avg=65536.0000 max=190797 max_rank=0 min=12 min_rank=5 imbalance_ratio=2.911331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] avg=65536.0000 max=66192 max_rank=0 min=63618 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=124605 max_load_reduction_pct=0.653076 moved_to_replicas=206569 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[33, 65603, 128386, 64327, 24, 6714, 62668, 196533] avg=65536.0000 max=196533 max_rank=7 min=24 min_rank=4 imbalance_ratio=2.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] avg=65536.0000 max=66192 max_rank=0 min=63618 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=124605 max_load_reduction_pct=0.653076 moved_to_replicas=206569 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] avg=65536.0000 max=66192 max_rank=0 min=63618 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=124605 max_load_reduction_pct=0.653076 moved_to_replicas=206569 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] avg=65536.0000 max=66192 max_rank=2 min=64327 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130341 max_load_reduction_pct=0.663202 moved_to_replicas=192535 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] avg=65536.0000 max=66192 max_rank=0 min=63618 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=124605 max_load_reduction_pct=0.653076 moved_to_replicas=206569 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[66192, 66192, 64923, 66192, 66192, 65521, 65458, 63618] avg=65536.0000 max=66192 max_rank=0 min=63618 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=124605 max_load_reduction_pct=0.653076 moved_to_replicas=206569 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[33, 65603, 128386, 64327, 24, 6714, 62668, 196533] avg=65536.0000 max=196533 max_rank=7 min=24 min_rank=4 imbalance_ratio=2.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] avg=65536.0000 max=66192 max_rank=2 min=64327 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130341 max_load_reduction_pct=0.663202 moved_to_replicas=192535 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[33, 65603, 128386, 64327, 24, 6714, 62668, 196533] avg=65536.0000 max=196533 max_rank=7 min=24 min_rank=4 imbalance_ratio=2.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[33, 65603, 128386, 64327, 24, 6714, 62668, 196533] avg=65536.0000 max=196533 max_rank=7 min=24 min_rank=4 imbalance_ratio=2.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[33, 65603, 128386, 64327, 24, 6714, 62668, 196533] avg=65536.0000 max=196533 max_rank=7 min=24 min_rank=4 imbalance_ratio=2.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[9, 50, 61191, 65577, 65714, 65453, 4225, 262069] avg=65536.0000 max=262069 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] avg=65536.0000 max=66192 max_rank=2 min=64327 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130341 max_load_reduction_pct=0.663202 moved_to_replicas=192535 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[33, 65603, 128386, 64327, 24, 6714, 62668, 196533] avg=65536.0000 max=196533 max_rank=7 min=24 min_rank=4 imbalance_ratio=2.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[33, 65603, 128386, 64327, 24, 6714, 62668, 196533] avg=65536.0000 max=196533 max_rank=7 min=24 min_rank=4 imbalance_ratio=2.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] avg=65536.0000 max=66192 max_rank=2 min=64327 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130341 max_load_reduction_pct=0.663202 moved_to_replicas=192535 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[9, 50, 61191, 65577, 65714, 65453, 4225, 262069] avg=65536.0000 max=262069 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] avg=65536.0000 max=66192 max_rank=2 min=64327 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130341 max_load_reduction_pct=0.663202 moved_to_replicas=192535 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] avg=65536.0000 max=66192 max_rank=6 min=64057 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195877 max_load_reduction_pct=0.747425 moved_to_replicas=195877 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] avg=65536.0000 max=66192 max_rank=2 min=64327 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130341 max_load_reduction_pct=0.663202 moved_to_replicas=192535 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] avg=65536.0000 max=66192 max_rank=2 min=64327 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130341 max_load_reduction_pct=0.663202 moved_to_replicas=192535 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[64849, 65603, 66192, 64327, 65549, 66192, 65384, 66192] avg=65536.0000 max=66192 max_rank=2 min=64327 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130341 max_load_reduction_pct=0.663202 moved_to_replicas=192535 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[9, 50, 61191, 65577, 65714, 65453, 4225, 262069] avg=65536.0000 max=262069 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] avg=65536.0000 max=66192 max_rank=6 min=64057 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195877 max_load_reduction_pct=0.747425 moved_to_replicas=195877 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[9, 50, 61191, 65577, 65714, 65453, 4225, 262069] avg=65536.0000 max=262069 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[197982, 65505, 68397, 58, 130085, 14, 188, 62059] avg=65536.0000 max=197982 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.020966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[9, 50, 61191, 65577, 65714, 65453, 4225, 262069] avg=65536.0000 max=262069 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[9, 50, 61191, 65577, 65714, 65453, 4225, 262069] avg=65536.0000 max=262069 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] avg=65536.0000 max=66192 max_rank=6 min=64057 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195877 max_load_reduction_pct=0.747425 moved_to_replicas=195877 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[9, 50, 61191, 65577, 65714, 65453, 4225, 262069] avg=65536.0000 max=262069 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[9, 50, 61191, 65577, 65714, 65453, 4225, 262069] avg=65536.0000 max=262069 max_rank=7 min=9 min_rank=0 imbalance_ratio=3.998856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[197982, 65505, 68397, 58, 130085, 14, 188, 62059] avg=65536.0000 max=197982 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.020966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] avg=65536.0000 max=66192 max_rank=6 min=64057 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195877 max_load_reduction_pct=0.747425 moved_to_replicas=195877 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] avg=65536.0000 max=66192 max_rank=0 min=64264 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131790 max_load_reduction_pct=0.665667 moved_to_replicas=197888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] avg=65536.0000 max=66192 max_rank=6 min=64057 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195877 max_load_reduction_pct=0.747425 moved_to_replicas=195877 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] avg=65536.0000 max=66192 max_rank=6 min=64057 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195877 max_load_reduction_pct=0.747425 moved_to_replicas=195877 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] avg=65536.0000 max=66192 max_rank=6 min=64057 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195877 max_load_reduction_pct=0.747425 moved_to_replicas=195877 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65532, 65571, 64057, 65577, 65714, 65453, 66192, 66192] avg=65536.0000 max=66192 max_rank=6 min=64057 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195877 max_load_reduction_pct=0.747425 moved_to_replicas=195877 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[197982, 65505, 68397, 58, 130085, 14, 188, 62059] avg=65536.0000 max=197982 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.020966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] avg=65536.0000 max=66192 max_rank=0 min=64264 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131790 max_load_reduction_pct=0.665667 moved_to_replicas=197888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65567, 17, 38328, 87817, 120718, 13, 146303, 65525] avg=65536.0000 max=146303 max_rank=6 min=13 min_rank=5 imbalance_ratio=2.232407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[197982, 65505, 68397, 58, 130085, 14, 188, 62059] avg=65536.0000 max=197982 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.020966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[197982, 65505, 68397, 58, 130085, 14, 188, 62059] avg=65536.0000 max=197982 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.020966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[197982, 65505, 68397, 58, 130085, 14, 188, 62059] avg=65536.0000 max=197982 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.020966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[197982, 65505, 68397, 58, 130085, 14, 188, 62059] avg=65536.0000 max=197982 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.020966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] avg=65536.0000 max=66192 max_rank=0 min=64264 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131790 max_load_reduction_pct=0.665667 moved_to_replicas=197888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] avg=65536.0000 max=66194 max_rank=6 min=62892 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=80109 max_load_reduction_pct=0.547555 moved_to_replicas=156262 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[197982, 65505, 68397, 58, 130085, 14, 188, 62059] avg=65536.0000 max=197982 max_rank=0 min=14 min_rank=5 imbalance_ratio=3.020966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] avg=65536.0000 max=66192 max_rank=0 min=64264 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131790 max_load_reduction_pct=0.665667 moved_to_replicas=197888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65567, 17, 38328, 87817, 120718, 13, 146303, 65525] avg=65536.0000 max=146303 max_rank=6 min=13 min_rank=5 imbalance_ratio=2.232407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] avg=65536.0000 max=66192 max_rank=0 min=64264 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131790 max_load_reduction_pct=0.665667 moved_to_replicas=197888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] avg=65536.0000 max=66192 max_rank=0 min=64264 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131790 max_load_reduction_pct=0.665667 moved_to_replicas=197888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] avg=65536.0000 max=66192 max_rank=0 min=64264 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131790 max_load_reduction_pct=0.665667 moved_to_replicas=197888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65567, 17, 38328, 87817, 120718, 13, 146303, 65525] avg=65536.0000 max=146303 max_rank=6 min=13 min_rank=5 imbalance_ratio=2.232407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[66192, 65505, 66192, 65575, 66192, 65534, 64834, 64264] avg=65536.0000 max=66192 max_rank=0 min=64264 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=131790 max_load_reduction_pct=0.665667 moved_to_replicas=197888 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] avg=65536.0000 max=66194 max_rank=6 min=62892 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=80109 max_load_reduction_pct=0.547555 moved_to_replicas=156262 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65536, 133257, 118, 65423, 65517, 65393, 65508, 63536] avg=65536.0000 max=133257 max_rank=1 min=118 min_rank=2 imbalance_ratio=2.033340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65567, 17, 38328, 87817, 120718, 13, 146303, 65525] avg=65536.0000 max=146303 max_rank=6 min=13 min_rank=5 imbalance_ratio=2.232407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65567, 17, 38328, 87817, 120718, 13, 146303, 65525] avg=65536.0000 max=146303 max_rank=6 min=13 min_rank=5 imbalance_ratio=2.232407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] avg=65536.0000 max=66194 max_rank=6 min=62892 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=80109 max_load_reduction_pct=0.547555 moved_to_replicas=156262 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65567, 17, 38328, 87817, 120718, 13, 146303, 65525] avg=65536.0000 max=146303 max_rank=6 min=13 min_rank=5 imbalance_ratio=2.232407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65567, 17, 38328, 87817, 120718, 13, 146303, 65525] avg=65536.0000 max=146303 max_rank=6 min=13 min_rank=5 imbalance_ratio=2.232407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] avg=65536.0000 max=66192 max_rank=1 min=65079 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67065 max_load_reduction_pct=0.503276 moved_to_replicas=67065 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] avg=65536.0000 max=66194 max_rank=6 min=62892 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=80109 max_load_reduction_pct=0.547555 moved_to_replicas=156262 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65567, 17, 38328, 87817, 120718, 13, 146303, 65525] avg=65536.0000 max=146303 max_rank=6 min=13 min_rank=5 imbalance_ratio=2.232407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65536, 133257, 118, 65423, 65517, 65393, 65508, 63536] avg=65536.0000 max=133257 max_rank=1 min=118 min_rank=2 imbalance_ratio=2.033340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] avg=65536.0000 max=66194 max_rank=6 min=62892 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=80109 max_load_reduction_pct=0.547555 moved_to_replicas=156262 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] avg=65536.0000 max=66194 max_rank=6 min=62892 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=80109 max_load_reduction_pct=0.547555 moved_to_replicas=156262 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] avg=65536.0000 max=66194 max_rank=6 min=62892 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=80109 max_load_reduction_pct=0.547555 moved_to_replicas=156262 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65536, 133257, 118, 65423, 65517, 65393, 65508, 63536] avg=65536.0000 max=133257 max_rank=1 min=118 min_rank=2 imbalance_ratio=2.033340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] avg=65536.0000 max=66192 max_rank=1 min=65079 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67065 max_load_reduction_pct=0.503276 moved_to_replicas=67065 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65567, 66190, 62892, 66193, 66192, 65535, 66194, 65525] avg=65536.0000 max=66194 max_rank=6 min=62892 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=80109 max_load_reduction_pct=0.547555 moved_to_replicas=156262 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[129814, 64565, 65535, 67596, 65538, 65524, 10, 65706] avg=65536.0000 max=129814 max_rank=0 min=10 min_rank=6 imbalance_ratio=1.980804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65536, 133257, 118, 65423, 65517, 65393, 65508, 63536] avg=65536.0000 max=133257 max_rank=1 min=118 min_rank=2 imbalance_ratio=2.033340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] avg=65536.0000 max=66192 max_rank=1 min=65079 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67065 max_load_reduction_pct=0.503276 moved_to_replicas=67065 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65536, 133257, 118, 65423, 65517, 65393, 65508, 63536] avg=65536.0000 max=133257 max_rank=1 min=118 min_rank=2 imbalance_ratio=2.033340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65536, 133257, 118, 65423, 65517, 65393, 65508, 63536] avg=65536.0000 max=133257 max_rank=1 min=118 min_rank=2 imbalance_ratio=2.033340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65536, 133257, 118, 65423, 65517, 65393, 65508, 63536] avg=65536.0000 max=133257 max_rank=1 min=118 min_rank=2 imbalance_ratio=2.033340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] avg=65536.0000 max=66192 max_rank=0 min=64565 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63622 max_load_reduction_pct=0.490101 moved_to_replicas=65026 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] avg=65536.0000 max=66192 max_rank=1 min=65079 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67065 max_load_reduction_pct=0.503276 moved_to_replicas=67065 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[129814, 64565, 65535, 67596, 65538, 65524, 10, 65706] avg=65536.0000 max=129814 max_rank=0 min=10 min_rank=6 imbalance_ratio=1.980804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65536, 133257, 118, 65423, 65517, 65393, 65508, 63536] avg=65536.0000 max=133257 max_rank=1 min=118 min_rank=2 imbalance_ratio=2.033340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] avg=65536.0000 max=66192 max_rank=1 min=65079 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67065 max_load_reduction_pct=0.503276 moved_to_replicas=67065 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] avg=65536.0000 max=66192 max_rank=1 min=65079 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67065 max_load_reduction_pct=0.503276 moved_to_replicas=67065 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] avg=65536.0000 max=66192 max_rank=1 min=65079 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67065 max_load_reduction_pct=0.503276 moved_to_replicas=67065 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[129814, 64565, 65535, 67596, 65538, 65524, 10, 65706] avg=65536.0000 max=129814 max_rank=0 min=10 min_rank=6 imbalance_ratio=1.980804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65544, 55, 65742, 138402, 65430, 65568, 57235, 66312] avg=65536.0000 max=138402 max_rank=3 min=55 min_rank=1 imbalance_ratio=2.111847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] avg=65536.0000 max=66192 max_rank=0 min=64565 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63622 max_load_reduction_pct=0.490101 moved_to_replicas=65026 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65536, 66192, 65640, 65423, 65517, 65393, 65508, 65079] avg=65536.0000 max=66192 max_rank=1 min=65079 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67065 max_load_reduction_pct=0.503276 moved_to_replicas=67065 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[129814, 64565, 65535, 67596, 65538, 65524, 10, 65706] avg=65536.0000 max=129814 max_rank=0 min=10 min_rank=6 imbalance_ratio=1.980804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] avg=65536.0000 max=66192 max_rank=0 min=64565 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63622 max_load_reduction_pct=0.490101 moved_to_replicas=65026 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[129814, 64565, 65535, 67596, 65538, 65524, 10, 65706] avg=65536.0000 max=129814 max_rank=0 min=10 min_rank=6 imbalance_ratio=1.980804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[129814, 64565, 65535, 67596, 65538, 65524, 10, 65706] avg=65536.0000 max=129814 max_rank=0 min=10 min_rank=6 imbalance_ratio=1.980804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[129814, 64565, 65535, 67596, 65538, 65524, 10, 65706] avg=65536.0000 max=129814 max_rank=0 min=10 min_rank=6 imbalance_ratio=1.980804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] avg=65536.0000 max=66193 max_rank=3 min=64087 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=72209 max_load_reduction_pct=0.521734 moved_to_replicas=72330 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] avg=65536.0000 max=66192 max_rank=0 min=64565 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63622 max_load_reduction_pct=0.490101 moved_to_replicas=65026 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65544, 55, 65742, 138402, 65430, 65568, 57235, 66312] avg=65536.0000 max=138402 max_rank=3 min=55 min_rank=1 imbalance_ratio=2.111847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] avg=65536.0000 max=66192 max_rank=0 min=64565 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63622 max_load_reduction_pct=0.490101 moved_to_replicas=65026 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[129814, 64565, 65535, 67596, 65538, 65524, 10, 65706] avg=65536.0000 max=129814 max_rank=0 min=10 min_rank=6 imbalance_ratio=1.980804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] avg=65536.0000 max=66192 max_rank=0 min=64565 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63622 max_load_reduction_pct=0.490101 moved_to_replicas=65026 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] avg=65536.0000 max=66192 max_rank=0 min=64565 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63622 max_load_reduction_pct=0.490101 moved_to_replicas=65026 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65544, 55, 65742, 138402, 65430, 65568, 57235, 66312] avg=65536.0000 max=138402 max_rank=3 min=55 min_rank=1 imbalance_ratio=2.111847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65543, 196335, 65517, 82, 560, 202, 65057, 130992] avg=65536.0000 max=196335 max_rank=1 min=82 min_rank=3 imbalance_ratio=2.995834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] avg=65536.0000 max=66193 max_rank=3 min=64087 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=72209 max_load_reduction_pct=0.521734 moved_to_replicas=72330 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[66192, 64565, 65535, 66192, 65538, 65524, 65036, 65706] avg=65536.0000 max=66192 max_rank=0 min=64565 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63622 max_load_reduction_pct=0.490101 moved_to_replicas=65026 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65544, 55, 65742, 138402, 65430, 65568, 57235, 66312] avg=65536.0000 max=138402 max_rank=3 min=55 min_rank=1 imbalance_ratio=2.111847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] avg=65536.0000 max=66193 max_rank=3 min=64087 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=72209 max_load_reduction_pct=0.521734 moved_to_replicas=72330 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] avg=65536.0000 max=66192 max_rank=1 min=64834 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130143 max_load_reduction_pct=0.662862 moved_to_replicas=194943 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65544, 55, 65742, 138402, 65430, 65568, 57235, 66312] avg=65536.0000 max=138402 max_rank=3 min=55 min_rank=1 imbalance_ratio=2.111847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65544, 55, 65742, 138402, 65430, 65568, 57235, 66312] avg=65536.0000 max=138402 max_rank=3 min=55 min_rank=1 imbalance_ratio=2.111847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65544, 55, 65742, 138402, 65430, 65568, 57235, 66312] avg=65536.0000 max=138402 max_rank=3 min=55 min_rank=1 imbalance_ratio=2.111847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] avg=65536.0000 max=66193 max_rank=3 min=64087 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=72209 max_load_reduction_pct=0.521734 moved_to_replicas=72330 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65543, 196335, 65517, 82, 560, 202, 65057, 130992] avg=65536.0000 max=196335 max_rank=1 min=82 min_rank=3 imbalance_ratio=2.995834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65544, 55, 65742, 138402, 65430, 65568, 57235, 66312] avg=65536.0000 max=138402 max_rank=3 min=55 min_rank=1 imbalance_ratio=2.111847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65543, 196335, 65517, 82, 560, 202, 65057, 130992] avg=65536.0000 max=196335 max_rank=1 min=82 min_rank=3 imbalance_ratio=2.995834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] avg=65536.0000 max=66193 max_rank=3 min=64087 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=72209 max_load_reduction_pct=0.521734 moved_to_replicas=72330 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] avg=65536.0000 max=66193 max_rank=3 min=64087 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=72209 max_load_reduction_pct=0.521734 moved_to_replicas=72330 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] avg=65536.0000 max=66193 max_rank=3 min=64087 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=72209 max_load_reduction_pct=0.521734 moved_to_replicas=72330 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[13, 133389, 92352, 65974, 51523, 131247, 49778, 12] avg=65536.0000 max=133389 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.035355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] avg=65536.0000 max=66192 max_rank=1 min=64834 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130143 max_load_reduction_pct=0.662862 moved_to_replicas=194943 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65544, 65532, 65742, 66193, 65430, 65568, 64087, 66192] avg=65536.0000 max=66193 max_rank=3 min=64087 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=72209 max_load_reduction_pct=0.521734 moved_to_replicas=72330 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] avg=65536.0000 max=66192 max_rank=1 min=64834 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130143 max_load_reduction_pct=0.662862 moved_to_replicas=194943 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65543, 196335, 65517, 82, 560, 202, 65057, 130992] avg=65536.0000 max=196335 max_rank=1 min=82 min_rank=3 imbalance_ratio=2.995834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] avg=65536.0000 max=66193 max_rank=6 min=61819 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=67196 max_load_reduction_pct=0.503760 moved_to_replicas=158412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65543, 196335, 65517, 82, 560, 202, 65057, 130992] avg=65536.0000 max=196335 max_rank=1 min=82 min_rank=3 imbalance_ratio=2.995834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65543, 196335, 65517, 82, 560, 202, 65057, 130992] avg=65536.0000 max=196335 max_rank=1 min=82 min_rank=3 imbalance_ratio=2.995834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65543, 196335, 65517, 82, 560, 202, 65057, 130992] avg=65536.0000 max=196335 max_rank=1 min=82 min_rank=3 imbalance_ratio=2.995834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] avg=65536.0000 max=66192 max_rank=1 min=64834 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130143 max_load_reduction_pct=0.662862 moved_to_replicas=194943 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[13, 133389, 92352, 65974, 51523, 131247, 49778, 12] avg=65536.0000 max=133389 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.035355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[13, 133389, 92352, 65974, 51523, 131247, 49778, 12] avg=65536.0000 max=133389 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.035355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] avg=65536.0000 max=66192 max_rank=1 min=64834 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130143 max_load_reduction_pct=0.662862 moved_to_replicas=194943 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65543, 196335, 65517, 82, 560, 202, 65057, 130992] avg=65536.0000 max=196335 max_rank=1 min=82 min_rank=3 imbalance_ratio=2.995834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] avg=65536.0000 max=66192 max_rank=1 min=64834 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130143 max_load_reduction_pct=0.662862 moved_to_replicas=194943 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] avg=65536.0000 max=66192 max_rank=1 min=64834 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130143 max_load_reduction_pct=0.662862 moved_to_replicas=194943 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[136176, 78615, 122610, 91, 72408, 46501, 52793, 15094] avg=65536.0000 max=136176 max_rank=0 min=91 min_rank=3 imbalance_ratio=2.077881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] avg=65536.0000 max=66193 max_rank=6 min=61819 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=67196 max_load_reduction_pct=0.503760 moved_to_replicas=158412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] avg=65536.0000 max=66193 max_rank=6 min=61819 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=67196 max_load_reduction_pct=0.503760 moved_to_replicas=158412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[13, 133389, 92352, 65974, 51523, 131247, 49778, 12] avg=65536.0000 max=133389 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.035355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65543, 66192, 65517, 65593, 65360, 64834, 65057, 66192] avg=65536.0000 max=66192 max_rank=1 min=64834 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130143 max_load_reduction_pct=0.662862 moved_to_replicas=194943 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] avg=65536.0000 max=66192 max_rank=0 min=62502 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69984 max_load_reduction_pct=0.513923 moved_to_replicas=145041 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[13, 133389, 92352, 65974, 51523, 131247, 49778, 12] avg=65536.0000 max=133389 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.035355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[13, 133389, 92352, 65974, 51523, 131247, 49778, 12] avg=65536.0000 max=133389 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.035355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[13, 133389, 92352, 65974, 51523, 131247, 49778, 12] avg=65536.0000 max=133389 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.035355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] avg=65536.0000 max=66193 max_rank=6 min=61819 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=67196 max_load_reduction_pct=0.503760 moved_to_replicas=158412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[136176, 78615, 122610, 91, 72408, 46501, 52793, 15094] avg=65536.0000 max=136176 max_rank=0 min=91 min_rank=3 imbalance_ratio=2.077881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[136176, 78615, 122610, 91, 72408, 46501, 52793, 15094] avg=65536.0000 max=136176 max_rank=0 min=91 min_rank=3 imbalance_ratio=2.077881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] avg=65536.0000 max=66193 max_rank=6 min=61819 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=67196 max_load_reduction_pct=0.503760 moved_to_replicas=158412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] avg=65536.0000 max=66193 max_rank=6 min=61819 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=67196 max_load_reduction_pct=0.503760 moved_to_replicas=158412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] avg=65536.0000 max=66193 max_rank=6 min=61819 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=67196 max_load_reduction_pct=0.503760 moved_to_replicas=158412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[13, 133389, 92352, 65974, 51523, 131247, 49778, 12] avg=65536.0000 max=133389 max_rank=1 min=12 min_rank=7 imbalance_ratio=2.035355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] avg=65536.0000 max=66192 max_rank=0 min=62502 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69984 max_load_reduction_pct=0.513923 moved_to_replicas=145041 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] avg=65536.0000 max=66192 max_rank=0 min=62502 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69984 max_load_reduction_pct=0.513923 moved_to_replicas=145041 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[136176, 78615, 122610, 91, 72408, 46501, 52793, 15094] avg=65536.0000 max=136176 max_rank=0 min=91 min_rank=3 imbalance_ratio=2.077881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 65974, 61819, 66192, 66193, 65534] avg=65536.0000 max=66193 max_rank=6 min=61819 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=67196 max_load_reduction_pct=0.503760 moved_to_replicas=158412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[136176, 78615, 122610, 91, 72408, 46501, 52793, 15094] avg=65536.0000 max=136176 max_rank=0 min=91 min_rank=3 imbalance_ratio=2.077881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[136176, 78615, 122610, 91, 72408, 46501, 52793, 15094] avg=65536.0000 max=136176 max_rank=0 min=91 min_rank=3 imbalance_ratio=2.077881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[136176, 78615, 122610, 91, 72408, 46501, 52793, 15094] avg=65536.0000 max=136176 max_rank=0 min=91 min_rank=3 imbalance_ratio=2.077881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] avg=65536.0000 max=66192 max_rank=0 min=62502 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69984 max_load_reduction_pct=0.513923 moved_to_replicas=145041 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] avg=65536.0000 max=66192 max_rank=0 min=62502 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69984 max_load_reduction_pct=0.513923 moved_to_replicas=145041 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] avg=65536.0000 max=66192 max_rank=0 min=62502 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69984 max_load_reduction_pct=0.513923 moved_to_replicas=145041 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] avg=65536.0000 max=66192 max_rank=0 min=62502 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69984 max_load_reduction_pct=0.513923 moved_to_replicas=145041 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[136176, 78615, 122610, 91, 72408, 46501, 52793, 15094] avg=65536.0000 max=136176 max_rank=0 min=91 min_rank=3 imbalance_ratio=2.077881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[66192, 66192, 66192, 65610, 66192, 62502, 65216, 66192] avg=65536.0000 max=66192 max_rank=0 min=62502 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=69984 max_load_reduction_pct=0.513923 moved_to_replicas=145041 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=12 total_expected_recv=785265 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191232 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=12 total_expected_recv=780103 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194048 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=12 total_expected_recv=784440 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190208 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=12 total_expected_recv=786878 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187264 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=12 total_expected_recv=790119 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183680 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=12 total_expected_recv=789722 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185088 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=12 total_expected_recv=785887 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189952 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=12 total_expected_recv=789042 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186112 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 15:44:42] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 341.2 | throughput per GPU (TFLOP/s/GPU): 529.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.437414E+00 | loss scale: 1.0 | grad norm: 5.193 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=81920 expected_recv=64814 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=81920 expected_recv=62321 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=81920 expected_recv=64814 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=81920 expected_recv=62321 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=81920 expected_recv=63413 actual_valid_recv=64896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65208 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=81920 expected_recv=64834 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=81920 expected_recv=63923 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65208 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=81920 expected_recv=63702 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=81920 expected_recv=64834 actual_valid_recv=65408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=81920 expected_recv=63923 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=81920 expected_recv=63702 actual_valid_recv=64000 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65094 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65191 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64928 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65625 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65021 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64928 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64589 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65021 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65094 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65191 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65021 expected_overflow=0 exact_recv_per_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64030 expected_overflow=0 exact_recv_per_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65021 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65632 expected_overflow=0 exact_recv_per_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64030 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65473 expected_overflow=0 exact_recv_per_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65632 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65473 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65625 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=81920 expected_recv=64589 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65556 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=66176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65663 expected_overflow=0 exact_recv_per_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=81920 expected_recv=64989 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65676 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=81920 expected_recv=64989 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=81920 expected_recv=63835 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65676 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=81920 expected_recv=63835 actual_valid_recv=64640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65572 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65663 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=81920 expected_recv=62793 expected_overflow=0 exact_recv_per_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=81920 expected_recv=62793 actual_valid_recv=63360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65451 expected_overflow=0 exact_recv_per_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=63360 slack_slots=18560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65451 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65612 expected_overflow=0 exact_recv_per_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65612 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65640 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64982 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64205 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65260 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64982 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64205 actual_valid_recv=64768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65724 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65260 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65443 expected_overflow=0 exact_recv_per_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65596 expected_overflow=0 exact_recv_per_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65443 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65596 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64163 expected_overflow=0 exact_recv_per_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64973 expected_overflow=0 exact_recv_per_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64163 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=81920 expected_recv=64973 actual_valid_recv=65536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65724 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64204 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64204 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64261 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64261 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64187 expected_overflow=0 exact_recv_per_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64187 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64238 expected_overflow=0 exact_recv_per_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=81920 expected_recv=64238 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65516 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=81920 expected_recv=64100 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65588 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65579 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=81920 expected_recv=64100 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=81920 expected_recv=62120 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=81920 expected_recv=62120 actual_valid_recv=62464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65577 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65630 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65588 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=81920 expected_recv=64921 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65491 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65612 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=81920 expected_recv=64921 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65491 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65612 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=81920 expected_recv=63727 expected_overflow=0 exact_recv_per_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=81920 expected_recv=63727 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65380 expected_overflow=0 exact_recv_per_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65380 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=81920 expected_recv=62911 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=81920 expected_recv=64138 expected_overflow=0 exact_recv_per_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=81920 expected_recv=62911 actual_valid_recv=63488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65559 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=81920 expected_recv=64138 actual_valid_recv=64384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=81920 expected_recv=64650 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=81920 expected_recv=63774 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65166 expected_overflow=0 exact_recv_per_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65655 expected_overflow=0 exact_recv_per_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65216 expected_overflow=0 exact_recv_per_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65166 actual_valid_recv=65664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=81920 expected_recv=64650 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=81920 expected_recv=63774 actual_valid_recv=64384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65655 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65216 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=81920 expected_recv=63047 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=81920 expected_recv=63047 actual_valid_recv=63488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=81920 expected_recv=62864 expected_overflow=0 exact_recv_per_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65528 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=81920 expected_recv=62864 actual_valid_recv=63488 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65389 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65389 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65556 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65559 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65568 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65498 actual_valid_recv=65664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=81920 expected_recv=64371 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=81920 expected_recv=64371 actual_valid_recv=64640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65419 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65259 expected_overflow=0 exact_recv_per_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65419 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=81920 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65259 actual_valid_recv=65792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=81920 expected_recv=63227 expected_overflow=0 exact_recv_per_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65356 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=81920 expected_recv=64872 actual_valid_recv=65152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65134 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65548 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65134 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=81920 expected_recv=63870 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=81920 expected_recv=63227 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65356 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65546 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=81920 expected_recv=63870 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=81920 expected_recv=63875 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=81920 expected_recv=63875 actual_valid_recv=64128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65621 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66018 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65582 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66018 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=65920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65582 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[36286, 121574, 79686, 21821, 164339, 25431, 32185, 42966] avg=65536.0000 max=164339 max_rank=4 min=21821 min_rank=3 imbalance_ratio=2.507614 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[36286, 121574, 79686, 21821, 164339, 25431, 32185, 42966] avg=65536.0000 max=164339 max_rank=4 min=21821 min_rank=3 imbalance_ratio=2.507614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[36286, 121574, 79686, 21821, 164339, 25431, 32185, 42966] avg=65536.0000 max=164339 max_rank=4 min=21821 min_rank=3 imbalance_ratio=2.507614 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[34525, 121448, 84551, 17880, 160596, 24755, 36088, 44445] avg=65536.0000 max=160596 max_rank=4 min=17880 min_rank=3 imbalance_ratio=2.450500 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[63413, 66192, 66193, 66192, 66192, 66193, 66191, 63722] avg=65536.0000 max=66193 max_rank=2 min=63413 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=94403 max_load_reduction_pct=0.587829 moved_to_replicas=168019 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[137385, 66630, 20545, 131073, 39923, 39, 63485, 65208] avg=65536.0000 max=137385 max_rank=0 min=39 min_rank=5 imbalance_ratio=2.096329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 66194, 66192, 64834, 65553, 63923, 65208] avg=65536.0000 max=66194 max_rank=2 min=63923 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=71191 max_load_reduction_pct=0.518186 moved_to_replicas=136512 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[17, 65625, 157813, 62, 986, 38513, 64589, 196683] avg=65536.0000 max=196683 max_rank=7 min=17 min_rank=0 imbalance_ratio=3.001144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[65549, 65625, 66192, 65021, 66192, 64928, 64589, 66192] avg=65536.0000 max=66192 max_rank=2 min=64589 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130491 max_load_reduction_pct=0.663458 moved_to_replicas=222112 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[112, 41, 8640, 65676, 117928, 64989, 5286, 261616] avg=65536.0000 max=261616 max_rank=7 min=41 min_rank=1 imbalance_ratio=3.991943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[65640, 65572, 63835, 65676, 66192, 64989, 66192, 66192] avg=65536.0000 max=66192 max_rank=4 min=63835 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=195424 max_load_reduction_pct=0.746988 moved_to_replicas=247160 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[198722, 65260, 1727, 63246, 130921, 12, 195, 64205] avg=65536.0000 max=198722 max_rank=0 min=12 min_rank=5 imbalance_ratio=3.032257 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[66192, 65260, 66192, 64982, 66192, 65541, 65724, 64205] avg=65536.0000 max=66192 max_rank=0 min=64205 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=132530 max_load_reduction_pct=0.666912 moved_to_replicas=197259 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65534, 8, 36262, 73307, 143623, 3, 140028, 65523] avg=65536.0000 max=143623 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.191513 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65534, 66191, 64261, 66192, 66192, 64204, 66191, 65523] avg=65536.0000 max=66192 max_rank=3 min=64204 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=77431 max_load_reduction_pct=0.539127 moved_to_replicas=158382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[109195, 132270, 46, 130523, 65630, 51150, 15707, 19767] avg=65536.0000 max=132270 max_rank=1 min=46 min_rank=2 imbalance_ratio=2.018280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 65577, 66192, 65630, 62120, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=62120 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66076 max_load_reduction_pct=0.499554 moved_to_replicas=173412 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[107561, 276, 65534, 94367, 65536, 65539, 4, 125471] avg=65536.0000 max=125471 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.914536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66193, 65534, 66191, 65536, 65539, 62911, 66192] avg=65536.0000 max=66193 max_rank=1 min=62911 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=59278 max_load_reduction_pct=0.472444 moved_to_replicas=128823 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65537, 25, 121456, 104132, 25601, 65559, 11161, 130817] avg=65536.0000 max=130817 max_rank=7 min=25 min_rank=1 imbalance_ratio=1.996109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65537, 64650, 66193, 66192, 63774, 65559, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=63774 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64624 max_load_reduction_pct=0.494003 moved_to_replicas=157829 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65556, 117704, 65528, 4221, 65389, 38, 81700, 124152] avg=65536.0000 max=124152 max_rank=7 min=38 min_rank=5 imbalance_ratio=1.894409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65556, 66192, 65528, 66192, 65389, 63047, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63047 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57960 max_load_reduction_pct=0.466847 moved_to_replicas=124980 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[6, 196589, 65515, 64371, 48224, 131071, 18507, 5] avg=65536.0000 max=196589 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.999710 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[64872, 66192, 65515, 64371, 65419, 66192, 66191, 65536] avg=65536.0000 max=66192 max_rank=1 min=64371 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=130397 max_load_reduction_pct=0.663298 moved_to_replicas=195276 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[194180, 131315, 65134, 19, 65621, 65534, 2458, 27] avg=65536.0000 max=194180 max_rank=0 min=19 min_rank=3 imbalance_ratio=2.962952 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65134, 65548, 65621, 65534, 66192, 63875] avg=65536.0000 max=66192 max_rank=0 min=63875 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=127988 max_load_reduction_pct=0.659120 moved_to_replicas=193111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=12 total_expected_recv=779399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=197248 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=12 total_expected_recv=788941 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187904 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=12 total_expected_recv=787163 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190336 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=12 total_expected_recv=785219 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190720 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=12 total_expected_recv=786348 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191232 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=12 total_expected_recv=790643 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=12 total_expected_recv=786681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=189952 peak_demand=66194 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=12 total_expected_recv=787062 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190464 peak_demand=66193 required_capacity=1.010025 - [2026-08-26 14:59:21] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 338.2 | throughput per GPU (TFLOP/s/GPU): 534.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.299762E+00 | loss scale: 1.0 | grad norm: 5.041 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=81920 expected_recv=64137 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=81920 expected_recv=62998 expected_overflow=0 exact_recv_per_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=81920 expected_recv=64137 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] avg=65536.0000 max=66194 max_rank=4 min=62321 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=98145 max_load_reduction_pct=0.597211 moved_to_replicas=167023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[36286, 121574, 79686, 21821, 164339, 25431, 32185, 42966] avg=65536.0000 max=164339 max_rank=4 min=21821 min_rank=3 imbalance_ratio=2.507614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[36286, 121574, 79686, 21821, 164339, 25431, 32185, 42966] avg=65536.0000 max=164339 max_rank=4 min=21821 min_rank=3 imbalance_ratio=2.507614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[36286, 121574, 79686, 21821, 164339, 25431, 32185, 42966] avg=65536.0000 max=164339 max_rank=4 min=21821 min_rank=3 imbalance_ratio=2.507614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] avg=65536.0000 max=66194 max_rank=4 min=62321 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=98145 max_load_reduction_pct=0.597211 moved_to_replicas=167023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] avg=65536.0000 max=66194 max_rank=4 min=62321 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=98145 max_load_reduction_pct=0.597211 moved_to_replicas=167023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[36286, 121574, 79686, 21821, 164339, 25431, 32185, 42966] avg=65536.0000 max=164339 max_rank=4 min=21821 min_rank=3 imbalance_ratio=2.507614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[36286, 121574, 79686, 21821, 164339, 25431, 32185, 42966] avg=65536.0000 max=164339 max_rank=4 min=21821 min_rank=3 imbalance_ratio=2.507614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] avg=65536.0000 max=66194 max_rank=4 min=62321 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=98145 max_load_reduction_pct=0.597211 moved_to_replicas=167023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[195373, 70735, 30, 131074, 59416, 24, 2542, 65094] avg=65536.0000 max=195373 max_rank=0 min=24 min_rank=5 imbalance_ratio=2.981155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] avg=65536.0000 max=66194 max_rank=4 min=62321 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=98145 max_load_reduction_pct=0.597211 moved_to_replicas=167023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] avg=65536.0000 max=66194 max_rank=4 min=62321 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=98145 max_load_reduction_pct=0.597211 moved_to_replicas=167023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[195373, 70735, 30, 131074, 59416, 24, 2542, 65094] avg=65536.0000 max=195373 max_rank=0 min=24 min_rank=5 imbalance_ratio=2.981155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] avg=65536.0000 max=66194 max_rank=4 min=62321 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=98145 max_load_reduction_pct=0.597211 moved_to_replicas=167023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[195373, 70735, 30, 131074, 59416, 24, 2542, 65094] avg=65536.0000 max=195373 max_rank=0 min=24 min_rank=5 imbalance_ratio=2.981155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66193, 66192, 66191, 66194, 62321, 64814, 66191] avg=65536.0000 max=66194 max_rank=4 min=62321 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=98145 max_load_reduction_pct=0.597211 moved_to_replicas=167023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] avg=65536.0000 max=66192 max_rank=0 min=63702 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=129181 max_load_reduction_pct=0.661202 moved_to_replicas=198606 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[195373, 70735, 30, 131074, 59416, 24, 2542, 65094] avg=65536.0000 max=195373 max_rank=0 min=24 min_rank=5 imbalance_ratio=2.981155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] avg=65536.0000 max=66192 max_rank=0 min=63702 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=129181 max_load_reduction_pct=0.661202 moved_to_replicas=198606 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[195373, 70735, 30, 131074, 59416, 24, 2542, 65094] avg=65536.0000 max=195373 max_rank=0 min=24 min_rank=5 imbalance_ratio=2.981155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[195373, 70735, 30, 131074, 59416, 24, 2542, 65094] avg=65536.0000 max=195373 max_rank=0 min=24 min_rank=5 imbalance_ratio=2.981155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] avg=65536.0000 max=66192 max_rank=0 min=63702 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=129181 max_load_reduction_pct=0.661202 moved_to_replicas=198606 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[195373, 70735, 30, 131074, 59416, 24, 2542, 65094] avg=65536.0000 max=195373 max_rank=0 min=24 min_rank=5 imbalance_ratio=2.981155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[24, 65632, 135212, 348, 7513, 65473, 53689, 196397] avg=65536.0000 max=196397 max_rank=7 min=24 min_rank=0 imbalance_ratio=2.996780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[195373, 70735, 30, 131074, 59416, 24, 2542, 65094] avg=65536.0000 max=195373 max_rank=0 min=24 min_rank=5 imbalance_ratio=2.981155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] avg=65536.0000 max=66192 max_rank=0 min=63702 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=129181 max_load_reduction_pct=0.661202 moved_to_replicas=198606 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] avg=65536.0000 max=66192 max_rank=0 min=63702 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=129181 max_load_reduction_pct=0.661202 moved_to_replicas=198606 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] avg=65536.0000 max=66192 max_rank=0 min=63702 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=129181 max_load_reduction_pct=0.661202 moved_to_replicas=198606 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[24, 65632, 135212, 348, 7513, 65473, 53689, 196397] avg=65536.0000 max=196397 max_rank=7 min=24 min_rank=0 imbalance_ratio=2.996780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] avg=65536.0000 max=66192 max_rank=2 min=64030 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130205 max_load_reduction_pct=0.662968 moved_to_replicas=199225 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] avg=65536.0000 max=66192 max_rank=0 min=63702 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=129181 max_load_reduction_pct=0.661202 moved_to_replicas=198606 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[24, 65632, 135212, 348, 7513, 65473, 53689, 196397] avg=65536.0000 max=196397 max_rank=7 min=24 min_rank=0 imbalance_ratio=2.996780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66192, 63702, 66192, 65191, 65533, 66192, 65094] avg=65536.0000 max=66192 max_rank=0 min=63702 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=129181 max_load_reduction_pct=0.661202 moved_to_replicas=198606 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] avg=65536.0000 max=66192 max_rank=2 min=64030 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130205 max_load_reduction_pct=0.662968 moved_to_replicas=199225 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[24, 65632, 135212, 348, 7513, 65473, 53689, 196397] avg=65536.0000 max=196397 max_rank=7 min=24 min_rank=0 imbalance_ratio=2.996780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[24, 65632, 135212, 348, 7513, 65473, 53689, 196397] avg=65536.0000 max=196397 max_rank=7 min=24 min_rank=0 imbalance_ratio=2.996780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[24, 65632, 135212, 348, 7513, 65473, 53689, 196397] avg=65536.0000 max=196397 max_rank=7 min=24 min_rank=0 imbalance_ratio=2.996780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] avg=65536.0000 max=66192 max_rank=2 min=64030 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130205 max_load_reduction_pct=0.662968 moved_to_replicas=199225 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[81, 885, 33018, 65663, 93066, 65451, 4053, 262071] avg=65536.0000 max=262071 max_rank=7 min=81 min_rank=0 imbalance_ratio=3.998886 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[24, 65632, 135212, 348, 7513, 65473, 53689, 196397] avg=65536.0000 max=196397 max_rank=7 min=24 min_rank=0 imbalance_ratio=2.996780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[24, 65632, 135212, 348, 7513, 65473, 53689, 196397] avg=65536.0000 max=196397 max_rank=7 min=24 min_rank=0 imbalance_ratio=2.996780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] avg=65536.0000 max=66192 max_rank=2 min=64030 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130205 max_load_reduction_pct=0.662968 moved_to_replicas=199225 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] avg=65536.0000 max=66192 max_rank=2 min=64030 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130205 max_load_reduction_pct=0.662968 moved_to_replicas=199225 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] avg=65536.0000 max=66192 max_rank=2 min=64030 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130205 max_load_reduction_pct=0.662968 moved_to_replicas=199225 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[81, 885, 33018, 65663, 93066, 65451, 4053, 262071] avg=65536.0000 max=262071 max_rank=7 min=81 min_rank=0 imbalance_ratio=3.998886 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=62793 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=195878 max_load_reduction_pct=0.747423 moved_to_replicas=222753 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[81, 885, 33018, 65663, 93066, 65451, 4053, 262071] avg=65536.0000 max=262071 max_rank=7 min=81 min_rank=0 imbalance_ratio=3.998886 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] avg=65536.0000 max=66192 max_rank=2 min=64030 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130205 max_load_reduction_pct=0.662968 moved_to_replicas=199225 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[65556, 65632, 66192, 65021, 66192, 65473, 64030, 66192] avg=65536.0000 max=66192 max_rank=2 min=64030 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130205 max_load_reduction_pct=0.662968 moved_to_replicas=199225 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=62793 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=195878 max_load_reduction_pct=0.747423 moved_to_replicas=222753 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[81, 885, 33018, 65663, 93066, 65451, 4053, 262071] avg=65536.0000 max=262071 max_rank=7 min=81 min_rank=0 imbalance_ratio=3.998886 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[81, 885, 33018, 65663, 93066, 65451, 4053, 262071] avg=65536.0000 max=262071 max_rank=7 min=81 min_rank=0 imbalance_ratio=3.998886 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[81, 885, 33018, 65663, 93066, 65451, 4053, 262071] avg=65536.0000 max=262071 max_rank=7 min=81 min_rank=0 imbalance_ratio=3.998886 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[197836, 64163, 130877, 169, 70365, 9, 69, 60800] avg=65536.0000 max=197836 max_rank=0 min=9 min_rank=5 imbalance_ratio=3.018738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=62793 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=195878 max_load_reduction_pct=0.747423 moved_to_replicas=222753 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[81, 885, 33018, 65663, 93066, 65451, 4053, 262071] avg=65536.0000 max=262071 max_rank=7 min=81 min_rank=0 imbalance_ratio=3.998886 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[81, 885, 33018, 65663, 93066, 65451, 4053, 262071] avg=65536.0000 max=262071 max_rank=7 min=81 min_rank=0 imbalance_ratio=3.998886 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=62793 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=195878 max_load_reduction_pct=0.747423 moved_to_replicas=222753 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=62793 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=195878 max_load_reduction_pct=0.747423 moved_to_replicas=222753 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=62793 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=195878 max_load_reduction_pct=0.747423 moved_to_replicas=222753 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[197836, 64163, 130877, 169, 70365, 9, 69, 60800] avg=65536.0000 max=197836 max_rank=0 min=9 min_rank=5 imbalance_ratio=3.018738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] avg=65536.0000 max=66192 max_rank=0 min=64163 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=131644 max_load_reduction_pct=0.665420 moved_to_replicas=200502 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[197836, 64163, 130877, 169, 70365, 9, 69, 60800] avg=65536.0000 max=197836 max_rank=0 min=9 min_rank=5 imbalance_ratio=3.018738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=62793 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=195878 max_load_reduction_pct=0.747423 moved_to_replicas=222753 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[65612, 66192, 62793, 65663, 66193, 65451, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=62793 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=195878 max_load_reduction_pct=0.747423 moved_to_replicas=222753 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] avg=65536.0000 max=66192 max_rank=0 min=64163 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=131644 max_load_reduction_pct=0.665420 moved_to_replicas=200502 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[197836, 64163, 130877, 169, 70365, 9, 69, 60800] avg=65536.0000 max=197836 max_rank=0 min=9 min_rank=5 imbalance_ratio=3.018738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] avg=65536.0000 max=66192 max_rank=0 min=64163 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=131644 max_load_reduction_pct=0.665420 moved_to_replicas=200502 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[197836, 64163, 130877, 169, 70365, 9, 69, 60800] avg=65536.0000 max=197836 max_rank=0 min=9 min_rank=5 imbalance_ratio=3.018738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[197836, 64163, 130877, 169, 70365, 9, 69, 60800] avg=65536.0000 max=197836 max_rank=0 min=9 min_rank=5 imbalance_ratio=3.018738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65579, 7, 20349, 64187, 168205, 3, 140442, 65516] avg=65536.0000 max=168205 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.566605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[197836, 64163, 130877, 169, 70365, 9, 69, 60800] avg=65536.0000 max=197836 max_rank=0 min=9 min_rank=5 imbalance_ratio=3.018738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[197836, 64163, 130877, 169, 70365, 9, 69, 60800] avg=65536.0000 max=197836 max_rank=0 min=9 min_rank=5 imbalance_ratio=3.018738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] avg=65536.0000 max=66192 max_rank=0 min=64163 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=131644 max_load_reduction_pct=0.665420 moved_to_replicas=200502 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] avg=65536.0000 max=66192 max_rank=0 min=64163 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=131644 max_load_reduction_pct=0.665420 moved_to_replicas=200502 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65579, 7, 20349, 64187, 168205, 3, 140442, 65516] avg=65536.0000 max=168205 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.566605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] avg=65536.0000 max=66193 max_rank=2 min=64187 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=102012 max_load_reduction_pct=0.606474 moved_to_replicas=176263 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] avg=65536.0000 max=66192 max_rank=0 min=64163 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=131644 max_load_reduction_pct=0.665420 moved_to_replicas=200502 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65579, 7, 20349, 64187, 168205, 3, 140442, 65516] avg=65536.0000 max=168205 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.566605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] avg=65536.0000 max=66192 max_rank=0 min=64163 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=131644 max_load_reduction_pct=0.665420 moved_to_replicas=200502 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[66192, 64163, 66192, 65443, 66192, 65537, 65596, 64973] avg=65536.0000 max=66192 max_rank=0 min=64163 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=131644 max_load_reduction_pct=0.665420 moved_to_replicas=200502 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] avg=65536.0000 max=66193 max_rank=2 min=64187 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=102012 max_load_reduction_pct=0.606474 moved_to_replicas=176263 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65579, 7, 20349, 64187, 168205, 3, 140442, 65516] avg=65536.0000 max=168205 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.566605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] avg=65536.0000 max=66193 max_rank=2 min=64187 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=102012 max_load_reduction_pct=0.606474 moved_to_replicas=176263 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65579, 7, 20349, 64187, 168205, 3, 140442, 65516] avg=65536.0000 max=168205 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.566605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[115087, 133721, 57, 64100, 65612, 64921, 76269, 4521] avg=65536.0000 max=133721 max_rank=1 min=57 min_rank=2 imbalance_ratio=2.040421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65579, 7, 20349, 64187, 168205, 3, 140442, 65516] avg=65536.0000 max=168205 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.566605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65579, 7, 20349, 64187, 168205, 3, 140442, 65516] avg=65536.0000 max=168205 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.566605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] avg=65536.0000 max=66193 max_rank=2 min=64187 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=102012 max_load_reduction_pct=0.606474 moved_to_replicas=176263 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65579, 7, 20349, 64187, 168205, 3, 140442, 65516] avg=65536.0000 max=168205 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.566605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[115087, 133721, 57, 64100, 65612, 64921, 76269, 4521] avg=65536.0000 max=133721 max_rank=1 min=57 min_rank=2 imbalance_ratio=2.040421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] avg=65536.0000 max=66193 max_rank=2 min=64187 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=102012 max_load_reduction_pct=0.606474 moved_to_replicas=176263 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] avg=65536.0000 max=66192 max_rank=0 min=64100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=67529 max_load_reduction_pct=0.504999 moved_to_replicas=126501 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] avg=65536.0000 max=66193 max_rank=2 min=64187 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=102012 max_load_reduction_pct=0.606474 moved_to_replicas=176263 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[115087, 133721, 57, 64100, 65612, 64921, 76269, 4521] avg=65536.0000 max=133721 max_rank=1 min=57 min_rank=2 imbalance_ratio=2.040421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] avg=65536.0000 max=66193 max_rank=2 min=64187 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=102012 max_load_reduction_pct=0.606474 moved_to_replicas=176263 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65579, 66191, 66193, 64187, 66192, 64238, 66192, 65516] avg=65536.0000 max=66193 max_rank=2 min=64187 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=102012 max_load_reduction_pct=0.606474 moved_to_replicas=176263 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] avg=65536.0000 max=66192 max_rank=0 min=64100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=67529 max_load_reduction_pct=0.504999 moved_to_replicas=126501 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[115087, 133721, 57, 64100, 65612, 64921, 76269, 4521] avg=65536.0000 max=133721 max_rank=1 min=57 min_rank=2 imbalance_ratio=2.040421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] avg=65536.0000 max=66192 max_rank=0 min=64100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=67529 max_load_reduction_pct=0.504999 moved_to_replicas=126501 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[98349, 65380, 65529, 51553, 65536, 65540, 4, 112397] avg=65536.0000 max=112397 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.715042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[115087, 133721, 57, 64100, 65612, 64921, 76269, 4521] avg=65536.0000 max=133721 max_rank=1 min=57 min_rank=2 imbalance_ratio=2.040421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[115087, 133721, 57, 64100, 65612, 64921, 76269, 4521] avg=65536.0000 max=133721 max_rank=1 min=57 min_rank=2 imbalance_ratio=2.040421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] avg=65536.0000 max=66192 max_rank=0 min=64100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=67529 max_load_reduction_pct=0.504999 moved_to_replicas=126501 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[115087, 133721, 57, 64100, 65612, 64921, 76269, 4521] avg=65536.0000 max=133721 max_rank=1 min=57 min_rank=2 imbalance_ratio=2.040421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[115087, 133721, 57, 64100, 65612, 64921, 76269, 4521] avg=65536.0000 max=133721 max_rank=1 min=57 min_rank=2 imbalance_ratio=2.040421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[98349, 65380, 65529, 51553, 65536, 65540, 4, 112397] avg=65536.0000 max=112397 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.715042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] avg=65536.0000 max=66192 max_rank=0 min=64100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=67529 max_load_reduction_pct=0.504999 moved_to_replicas=126501 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] avg=65536.0000 max=66193 max_rank=0 min=63727 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=46204 max_load_reduction_pct=0.411079 moved_to_replicas=78362 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] avg=65536.0000 max=66192 max_rank=0 min=64100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=67529 max_load_reduction_pct=0.504999 moved_to_replicas=126501 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[98349, 65380, 65529, 51553, 65536, 65540, 4, 112397] avg=65536.0000 max=112397 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.715042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] avg=65536.0000 max=66192 max_rank=0 min=64100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=67529 max_load_reduction_pct=0.504999 moved_to_replicas=126501 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] avg=65536.0000 max=66193 max_rank=0 min=63727 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=46204 max_load_reduction_pct=0.411079 moved_to_replicas=78362 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[66192, 66192, 65588, 64100, 65612, 64921, 66192, 65491] avg=65536.0000 max=66192 max_rank=0 min=64100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=67529 max_load_reduction_pct=0.504999 moved_to_replicas=126501 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[98349, 65380, 65529, 51553, 65536, 65540, 4, 112397] avg=65536.0000 max=112397 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.715042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] avg=65536.0000 max=66193 max_rank=0 min=63727 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=46204 max_load_reduction_pct=0.411079 moved_to_replicas=78362 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[98349, 65380, 65529, 51553, 65536, 65540, 4, 112397] avg=65536.0000 max=112397 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.715042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65537, 10, 127389, 67515, 65216, 65655, 2941, 130025] avg=65536.0000 max=130025 max_rank=7 min=10 min_rank=1 imbalance_ratio=1.984024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[98349, 65380, 65529, 51553, 65536, 65540, 4, 112397] avg=65536.0000 max=112397 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.715042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] avg=65536.0000 max=66193 max_rank=0 min=63727 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=46204 max_load_reduction_pct=0.411079 moved_to_replicas=78362 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65537, 10, 127389, 67515, 65216, 65655, 2941, 130025] avg=65536.0000 max=130025 max_rank=7 min=10 min_rank=1 imbalance_ratio=1.984024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[98349, 65380, 65529, 51553, 65536, 65540, 4, 112397] avg=65536.0000 max=112397 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.715042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[98349, 65380, 65529, 51553, 65536, 65540, 4, 112397] avg=65536.0000 max=112397 max_rank=7 min=4 min_rank=6 imbalance_ratio=1.715042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] avg=65536.0000 max=66193 max_rank=0 min=63727 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=46204 max_load_reduction_pct=0.411079 moved_to_replicas=78362 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] avg=65536.0000 max=66192 max_rank=2 min=64138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63833 max_load_reduction_pct=0.490929 moved_to_replicas=126353 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] avg=65536.0000 max=66193 max_rank=0 min=63727 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=46204 max_load_reduction_pct=0.411079 moved_to_replicas=78362 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65537, 10, 127389, 67515, 65216, 65655, 2941, 130025] avg=65536.0000 max=130025 max_rank=7 min=10 min_rank=1 imbalance_ratio=1.984024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] avg=65536.0000 max=66192 max_rank=2 min=64138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63833 max_load_reduction_pct=0.490929 moved_to_replicas=126353 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65537, 10, 127389, 67515, 65216, 65655, 2941, 130025] avg=65536.0000 max=130025 max_rank=7 min=10 min_rank=1 imbalance_ratio=1.984024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] avg=65536.0000 max=66193 max_rank=0 min=63727 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=46204 max_load_reduction_pct=0.411079 moved_to_replicas=78362 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[66193, 65380, 65529, 63727, 65536, 65540, 66190, 66193] avg=65536.0000 max=66193 max_rank=0 min=63727 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=46204 max_load_reduction_pct=0.411079 moved_to_replicas=78362 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] avg=65536.0000 max=66192 max_rank=2 min=64138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63833 max_load_reduction_pct=0.490929 moved_to_replicas=126353 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65568, 146768, 65529, 48467, 5614, 42, 61269, 131031] avg=65536.0000 max=146768 max_rank=1 min=42 min_rank=5 imbalance_ratio=2.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65537, 10, 127389, 67515, 65216, 65655, 2941, 130025] avg=65536.0000 max=130025 max_rank=7 min=10 min_rank=1 imbalance_ratio=1.984024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65537, 10, 127389, 67515, 65216, 65655, 2941, 130025] avg=65536.0000 max=130025 max_rank=7 min=10 min_rank=1 imbalance_ratio=1.984024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] avg=65536.0000 max=66192 max_rank=2 min=64138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63833 max_load_reduction_pct=0.490929 moved_to_replicas=126353 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65568, 146768, 65529, 48467, 5614, 42, 61269, 131031] avg=65536.0000 max=146768 max_rank=1 min=42 min_rank=5 imbalance_ratio=2.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65537, 10, 127389, 67515, 65216, 65655, 2941, 130025] avg=65536.0000 max=130025 max_rank=7 min=10 min_rank=1 imbalance_ratio=1.984024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65537, 10, 127389, 67515, 65216, 65655, 2941, 130025] avg=65536.0000 max=130025 max_rank=7 min=10 min_rank=1 imbalance_ratio=1.984024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] avg=65536.0000 max=66193 max_rank=4 min=62864 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=80575 max_load_reduction_pct=0.548996 moved_to_replicas=145415 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65568, 146768, 65529, 48467, 5614, 42, 61269, 131031] avg=65536.0000 max=146768 max_rank=1 min=42 min_rank=5 imbalance_ratio=2.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] avg=65536.0000 max=66192 max_rank=2 min=64138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63833 max_load_reduction_pct=0.490929 moved_to_replicas=126353 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] avg=65536.0000 max=66192 max_rank=2 min=64138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63833 max_load_reduction_pct=0.490929 moved_to_replicas=126353 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] avg=65536.0000 max=66193 max_rank=4 min=62864 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=80575 max_load_reduction_pct=0.548996 moved_to_replicas=145415 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65568, 146768, 65529, 48467, 5614, 42, 61269, 131031] avg=65536.0000 max=146768 max_rank=1 min=42 min_rank=5 imbalance_ratio=2.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] avg=65536.0000 max=66192 max_rank=2 min=64138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63833 max_load_reduction_pct=0.490929 moved_to_replicas=126353 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65537, 65166, 66192, 66192, 65216, 65655, 64138, 66192] avg=65536.0000 max=66192 max_rank=2 min=64138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63833 max_load_reduction_pct=0.490929 moved_to_replicas=126353 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] avg=65536.0000 max=66193 max_rank=4 min=62864 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=80575 max_load_reduction_pct=0.548996 moved_to_replicas=145415 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[7, 191406, 65498, 69729, 46380, 131403, 19860, 5] avg=65536.0000 max=191406 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65568, 146768, 65529, 48467, 5614, 42, 61269, 131031] avg=65536.0000 max=146768 max_rank=1 min=42 min_rank=5 imbalance_ratio=2.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] avg=65536.0000 max=66193 max_rank=4 min=62864 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=80575 max_load_reduction_pct=0.548996 moved_to_replicas=145415 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65568, 146768, 65529, 48467, 5614, 42, 61269, 131031] avg=65536.0000 max=146768 max_rank=1 min=42 min_rank=5 imbalance_ratio=2.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[7, 191406, 65498, 69729, 46380, 131403, 19860, 5] avg=65536.0000 max=191406 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=1 min=63227 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=125214 max_load_reduction_pct=0.654180 moved_to_replicas=193962 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65568, 146768, 65529, 48467, 5614, 42, 61269, 131031] avg=65536.0000 max=146768 max_rank=1 min=42 min_rank=5 imbalance_ratio=2.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65568, 146768, 65529, 48467, 5614, 42, 61269, 131031] avg=65536.0000 max=146768 max_rank=1 min=42 min_rank=5 imbalance_ratio=2.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[7, 191406, 65498, 69729, 46380, 131403, 19860, 5] avg=65536.0000 max=191406 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] avg=65536.0000 max=66193 max_rank=4 min=62864 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=80575 max_load_reduction_pct=0.548996 moved_to_replicas=145415 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] avg=65536.0000 max=66193 max_rank=4 min=62864 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=80575 max_load_reduction_pct=0.548996 moved_to_replicas=145415 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=1 min=63227 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=125214 max_load_reduction_pct=0.654180 moved_to_replicas=193962 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[7, 191406, 65498, 69729, 46380, 131403, 19860, 5] avg=65536.0000 max=191406 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] avg=65536.0000 max=66193 max_rank=4 min=62864 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=80575 max_load_reduction_pct=0.548996 moved_to_replicas=145415 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65568, 66191, 65529, 66192, 66193, 65559, 62864, 66192] avg=65536.0000 max=66193 max_rank=4 min=62864 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=80575 max_load_reduction_pct=0.548996 moved_to_replicas=145415 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=1 min=63227 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=125214 max_load_reduction_pct=0.654180 moved_to_replicas=193962 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[132696, 131167, 65356, 17, 65582, 65532, 63870, 68] avg=65536.0000 max=132696 max_rank=0 min=17 min_rank=3 imbalance_ratio=2.024780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=1 min=63227 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=125214 max_load_reduction_pct=0.654180 moved_to_replicas=193962 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[132696, 131167, 65356, 17, 65582, 65532, 63870, 68] avg=65536.0000 max=132696 max_rank=0 min=17 min_rank=3 imbalance_ratio=2.024780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[7, 191406, 65498, 69729, 46380, 131403, 19860, 5] avg=65536.0000 max=191406 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[7, 191406, 65498, 69729, 46380, 131403, 19860, 5] avg=65536.0000 max=191406 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] avg=65536.0000 max=66192 max_rank=0 min=63870 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66504 max_load_reduction_pct=0.501176 moved_to_replicas=131479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[7, 191406, 65498, 69729, 46380, 131403, 19860, 5] avg=65536.0000 max=191406 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[132696, 131167, 65356, 17, 65582, 65532, 63870, 68] avg=65536.0000 max=132696 max_rank=0 min=17 min_rank=3 imbalance_ratio=2.024780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[7, 191406, 65498, 69729, 46380, 131403, 19860, 5] avg=65536.0000 max=191406 max_rank=1 min=5 min_rank=7 imbalance_ratio=2.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] avg=65536.0000 max=66192 max_rank=0 min=63870 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66504 max_load_reduction_pct=0.501176 moved_to_replicas=131479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=1 min=63227 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=125214 max_load_reduction_pct=0.654180 moved_to_replicas=193962 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=1 min=63227 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=125214 max_load_reduction_pct=0.654180 moved_to_replicas=193962 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[132696, 131167, 65356, 17, 65582, 65532, 63870, 68] avg=65536.0000 max=132696 max_rank=0 min=17 min_rank=3 imbalance_ratio=2.024780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=1 min=63227 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=125214 max_load_reduction_pct=0.654180 moved_to_replicas=193962 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] avg=65536.0000 max=66192 max_rank=0 min=63870 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66504 max_load_reduction_pct=0.501176 moved_to_replicas=131479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[63227, 66192, 65498, 66192, 65259, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=1 min=63227 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=125214 max_load_reduction_pct=0.654180 moved_to_replicas=193962 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] avg=65536.0000 max=66192 max_rank=0 min=63870 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66504 max_load_reduction_pct=0.501176 moved_to_replicas=131479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[132696, 131167, 65356, 17, 65582, 65532, 63870, 68] avg=65536.0000 max=132696 max_rank=0 min=17 min_rank=3 imbalance_ratio=2.024780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[132696, 131167, 65356, 17, 65582, 65532, 63870, 68] avg=65536.0000 max=132696 max_rank=0 min=17 min_rank=3 imbalance_ratio=2.024780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[132696, 131167, 65356, 17, 65582, 65532, 63870, 68] avg=65536.0000 max=132696 max_rank=0 min=17 min_rank=3 imbalance_ratio=2.024780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[132696, 131167, 65356, 17, 65582, 65532, 63870, 68] avg=65536.0000 max=132696 max_rank=0 min=17 min_rank=3 imbalance_ratio=2.024780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] avg=65536.0000 max=66192 max_rank=0 min=63870 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66504 max_load_reduction_pct=0.501176 moved_to_replicas=131479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] avg=65536.0000 max=66192 max_rank=0 min=63870 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66504 max_load_reduction_pct=0.501176 moved_to_replicas=131479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] avg=65536.0000 max=66192 max_rank=0 min=63870 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66504 max_load_reduction_pct=0.501176 moved_to_replicas=131479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[66192, 66192, 65356, 65546, 65582, 65532, 63870, 66018] avg=65536.0000 max=66192 max_rank=0 min=63870 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66504 max_load_reduction_pct=0.501176 moved_to_replicas=131479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=12 total_expected_recv=784956 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192512 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=12 total_expected_recv=782462 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194432 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=12 total_expected_recv=784646 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=191616 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=12 total_expected_recv=781952 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194560 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=12 total_expected_recv=789780 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186752 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=12 total_expected_recv=789876 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185600 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=12 total_expected_recv=789552 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187648 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=12 total_expected_recv=788232 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188672 peak_demand=66193 required_capacity=1.010025 + [2026-08-26 15:44:42] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 339.8 | throughput per GPU (TFLOP/s/GPU): 532.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.310550E+00 | loss scale: 1.0 | grad norm: 4.907 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=81920 expected_recv=61175 expected_overflow=0 exact_recv_per_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65960 expected_overflow=0 exact_recv_per_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=81920 expected_recv=61175 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67712 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65960 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67328 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67584 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=81920 expected_recv=62998 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65329 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66002 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65329 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66002 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=81920 expected_recv=61997 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67584 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65165 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=81920 expected_recv=64905 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65165 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=81920 expected_recv=64905 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=81920 expected_recv=63258 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=81920 expected_recv=61997 actual_valid_recv=62592 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=81920 expected_recv=63258 actual_valid_recv=63616 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65231 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65231 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65715 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=81920 expected_recv=62985 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65715 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=81920 expected_recv=62985 actual_valid_recv=63360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65669 expected_overflow=0 exact_recv_per_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65669 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65589 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=63360 slack_slots=18560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=81920 expected_recv=64543 expected_overflow=0 exact_recv_per_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=81920 expected_recv=64443 expected_overflow=0 exact_recv_per_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65506 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=81920 expected_recv=64543 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=81920 expected_recv=64443 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=81920 expected_recv=64500 expected_overflow=0 exact_recv_per_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=81920 expected_recv=64500 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=81920 expected_recv=64155 expected_overflow=0 exact_recv_per_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=81920 expected_recv=61725 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65411 expected_overflow=0 exact_recv_per_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65505 expected_overflow=0 exact_recv_per_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65359 expected_overflow=0 exact_recv_per_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=81920 expected_recv=64155 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65505 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65359 actual_valid_recv=65536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=81920 expected_recv=61725 actual_valid_recv=62464 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65411 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=62464 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=81920 expected_recv=61455 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=81920 expected_recv=61455 actual_valid_recv=62080 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65681 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=62080 slack_slots=19840 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65681 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65499 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=81920 expected_recv=62607 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65222 expected_overflow=0 exact_recv_per_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=81920 expected_recv=62607 actual_valid_recv=62976 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65502 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65551 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=81920 expected_recv=62897 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=81920 expected_recv=62897 actual_valid_recv=63232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65570 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=63232 slack_slots=18688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65456 actual_valid_recv=65792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65222 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65042 expected_overflow=0 exact_recv_per_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63882 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65042 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=81920 expected_recv=63524 expected_overflow=0 exact_recv_per_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=81920 expected_recv=63524 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65498 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65658 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=81920 expected_recv=64168 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63502 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=81920 expected_recv=64168 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63502 actual_valid_recv=64640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63882 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65183 expected_overflow=0 exact_recv_per_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65613 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63837 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=81920 expected_recv=63837 actual_valid_recv=64128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=81920 expected_recv=63479 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=81920 expected_recv=63479 actual_valid_recv=64256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=81920 expected_recv=62646 expected_overflow=0 exact_recv_per_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65183 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=81920 expected_recv=62646 actual_valid_recv=63104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65500 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=63104 slack_slots=18816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=81920 expected_recv=64336 expected_overflow=0 exact_recv_per_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=81920 expected_recv=61840 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=81920 expected_recv=64336 actual_valid_recv=64512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=81920 expected_recv=61840 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65933 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65557 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65933 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65569 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=81920 expected_recv=61567 expected_overflow=0 exact_recv_per_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65094 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65557 actual_valid_recv=65920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65688 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65094 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65450 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65688 actual_valid_recv=66176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65450 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=81920 expected_recv=61567 actual_valid_recv=62080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65486 expected_overflow=0 exact_recv_per_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=81920 expected_recv=63642 expected_overflow=0 exact_recv_per_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=62080 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65486 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=81920 expected_recv=63642 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=81920 expected_recv=63822 expected_overflow=0 exact_recv_per_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64183 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65658 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65032 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=81920 expected_recv=63822 actual_valid_recv=64512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64183 actual_valid_recv=64640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64772 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65032 actual_valid_recv=65280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64772 actual_valid_recv=64896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=81920 expected_recv=64671 expected_overflow=0 exact_recv_per_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65721 expected_overflow=0 exact_recv_per_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=81920 expected_recv=64671 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65721 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64376 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65853 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=81920 expected_recv=64376 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65853 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65288 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=81920 expected_recv=64914 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65572 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65288 actual_valid_recv=65536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=81920 expected_recv=64914 actual_valid_recv=65024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65075 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65075 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=81920 expected_recv=63757 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=81920 expected_recv=64404 expected_overflow=0 exact_recv_per_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=81920 expected_recv=63757 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=81920 expected_recv=64404 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65460 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=65792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[30637, 124041, 76933, 17046, 164058, 27165, 35390, 49018] avg=65536.0000 max=164058 max_rank=4 min=17046 min_rank=3 imbalance_ratio=2.503326 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[62998, 66192, 66192, 66192, 66193, 66192, 66192, 64137] avg=65536.0000 max=66193 max_rank=4 min=62998 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=97865 max_load_reduction_pct=0.596527 moved_to_replicas=166456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[66649, 66002, 8442, 131073, 65329, 57475, 22779, 106539] avg=65536.0000 max=131073 max_rank=3 min=8442 min_rank=2 imbalance_ratio=2.000015 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[66192, 66002, 66192, 66192, 65329, 61997, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=61997 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64881 max_load_reduction_pct=0.494999 moved_to_replicas=105685 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[60, 65715, 141114, 374, 65231, 3864, 54587, 193343] avg=65536.0000 max=193343 max_rank=7 min=60 min_rank=0 imbalance_ratio=2.950180 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[65589, 65715, 66192, 62985, 65231, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62985 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=127151 max_load_reduction_pct=0.657645 moved_to_replicas=202073 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65411, 45390, 25913, 68569, 68080, 48921, 5483, 196521] avg=65536.0000 max=196521 max_rank=7 min=5483 min_rank=6 imbalance_ratio=2.998672 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65411, 66192, 66192, 66192, 66192, 61725, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=130329 max_load_reduction_pct=0.663181 moved_to_replicas=134594 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[198963, 22460, 36583, 67731, 129558, 10851, 154, 57988] avg=65536.0000 max=198963 max_rank=0 min=154 min_rank=6 imbalance_ratio=3.035934 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[66192, 66193, 66192, 66192, 66192, 66191, 65681, 61455] avg=65536.0000 max=66193 max_rank=1 min=61455 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=132770 max_load_reduction_pct=0.667310 moved_to_replicas=197676 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65222, 23, 56470, 128591, 110496, 15, 97972, 65499] avg=65536.0000 max=128591 max_rank=3 min=15 min_rank=5 imbalance_ratio=1.962143 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65222, 66192, 66192, 66192, 66192, 62607, 66192, 65499] avg=65536.0000 max=66192 max_rank=1 min=62607 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62399 max_load_reduction_pct=0.485252 moved_to_replicas=138483 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[68641, 168854, 84, 127831, 66447, 28757, 63582, 92] avg=65536.0000 max=168854 max_rank=1 min=84 min_rank=2 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66190, 65613, 66192, 66192, 66190, 63837, 63882] avg=65536.0000 max=66192 max_rank=0 min=63837 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102662 max_load_reduction_pct=0.607993 moved_to_replicas=167005 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[73651, 61, 65513, 68522, 130526, 67837, 2, 118176] avg=65536.0000 max=130526 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.991669 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[66192, 63479, 65513, 66192, 66192, 66192, 64336, 66192] avg=65536.0000 max=66192 max_rank=0 min=63479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64334 max_load_reduction_pct=0.492883 moved_to_replicas=127752 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65557, 14623, 126898, 54527, 57750, 70282, 4407, 130244] avg=65536.0000 max=130244 max_rank=7 min=4407 min_rank=6 imbalance_ratio=1.987366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65557, 66190, 66192, 65933, 61840, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61840 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64052 max_load_reduction_pct=0.491785 moved_to_replicas=128848 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65658, 75042, 65450, 126288, 65094, 24, 61044, 65688] avg=65536.0000 max=126288 max_rank=3 min=24 min_rank=5 imbalance_ratio=1.927002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65658, 66192, 65450, 66192, 65094, 66192, 63822, 65688] avg=65536.0000 max=66192 max_rank=1 min=63822 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60096 max_load_reduction_pct=0.475865 moved_to_replicas=68946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[6, 196742, 64772, 114265, 11904, 131066, 5523, 10] avg=65536.0000 max=196742 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.002045 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64772, 66191, 64183, 66192, 66192, 65032] avg=65536.0000 max=66192 max_rank=1 min=64183 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=130550 max_load_reduction_pct=0.663559 moved_to_replicas=243497 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[131040, 131024, 65288, 243, 65572, 65526, 65529, 66] avg=65536.0000 max=131040 max_rank=0 min=66 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66192, 65288, 65075, 65572, 65526, 65529, 64914] avg=65536.0000 max=66192 max_rank=0 min=64914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64848 max_load_reduction_pct=0.494872 moved_to_replicas=129680 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=12 total_expected_recv=784402 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=192640 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=12 total_expected_recv=781567 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194304 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=12 total_expected_recv=790921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=66193 required_capacity=1.010025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=12 total_expected_recv=781388 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=195456 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=12 total_expected_recv=789720 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187008 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=12 total_expected_recv=789980 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187264 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=12 total_expected_recv=786929 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190080 peak_demand=66192 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=12 total_expected_recv=786549 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190080 peak_demand=66192 required_capacity=1.010010 - [2026-08-26 14:59:22] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 338.1 | throughput per GPU (TFLOP/s/GPU): 534.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.177852E+00 | loss scale: 1.0 | grad norm: 4.691 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | -[after training is done] datetime: 2026-08-26 14:59:22 -W0826 14:59:23.077000 44602 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 14:59:23.078000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44695 closing signal SIGTERM -W0826 14:59:23.078000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44696 closing signal SIGTERM -W0826 14:59:23.079000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44697 closing signal SIGTERM -W0826 14:59:23.080000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44698 closing signal SIGTERM -W0826 14:59:23.081000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44699 closing signal SIGTERM -W0826 14:59:23.082000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44700 closing signal SIGTERM -W0826 14:59:23.082000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44701 closing signal SIGTERM -W0826 14:59:23.083000 44602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 44702 closing signal SIGTERM +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=81920 expected_recv=64744 expected_overflow=0 exact_recv_per_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=81920 expected_recv=66190 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=81920 expected_recv=64744 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32400, 125217, 71873, 22567, 165257, 28108, 32626, 46240] avg=65536.0000 max=165257 max_rank=4 min=22567 min_rank=3 imbalance_ratio=2.521622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32400, 125217, 71873, 22567, 165257, 28108, 32626, 46240] avg=65536.0000 max=165257 max_rank=4 min=22567 min_rank=3 imbalance_ratio=2.521622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32400, 125217, 71873, 22567, 165257, 28108, 32626, 46240] avg=65536.0000 max=165257 max_rank=4 min=22567 min_rank=3 imbalance_ratio=2.521622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65580 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32400, 125217, 71873, 22567, 165257, 28108, 32626, 46240] avg=65536.0000 max=165257 max_rank=4 min=22567 min_rank=3 imbalance_ratio=2.521622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32400, 125217, 71873, 22567, 165257, 28108, 32626, 46240] avg=65536.0000 max=165257 max_rank=4 min=22567 min_rank=3 imbalance_ratio=2.521622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=61175 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=99064 max_load_reduction_pct=0.599454 moved_to_replicas=163771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=61175 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=99064 max_load_reduction_pct=0.599454 moved_to_replicas=163771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=61175 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=99064 max_load_reduction_pct=0.599454 moved_to_replicas=163771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=61175 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=99064 max_load_reduction_pct=0.599454 moved_to_replicas=163771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=61175 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=99064 max_load_reduction_pct=0.599454 moved_to_replicas=163771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32400, 125217, 71873, 22567, 165257, 28108, 32626, 46240] avg=65536.0000 max=165257 max_rank=4 min=22567 min_rank=3 imbalance_ratio=2.521622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32400, 125217, 71873, 22567, 165257, 28108, 32626, 46240] avg=65536.0000 max=165257 max_rank=4 min=22567 min_rank=3 imbalance_ratio=2.521622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32400, 125217, 71873, 22567, 165257, 28108, 32626, 46240] avg=65536.0000 max=165257 max_rank=4 min=22567 min_rank=3 imbalance_ratio=2.521622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[115102, 73743, 32, 131065, 62724, 5320, 65165, 71137] avg=65536.0000 max=131065 max_rank=3 min=32 min_rank=2 imbalance_ratio=1.999893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[115102, 73743, 32, 131065, 62724, 5320, 65165, 71137] avg=65536.0000 max=131065 max_rank=3 min=32 min_rank=2 imbalance_ratio=1.999893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[115102, 73743, 32, 131065, 62724, 5320, 65165, 71137] avg=65536.0000 max=131065 max_rank=3 min=32 min_rank=2 imbalance_ratio=1.999893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=61175 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=99064 max_load_reduction_pct=0.599454 moved_to_replicas=163771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=61175 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=99064 max_load_reduction_pct=0.599454 moved_to_replicas=163771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[115102, 73743, 32, 131065, 62724, 5320, 65165, 71137] avg=65536.0000 max=131065 max_rank=3 min=32 min_rank=2 imbalance_ratio=1.999893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[65960, 66192, 66192, 66193, 66192, 61175, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=61175 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=99064 max_load_reduction_pct=0.599454 moved_to_replicas=163771 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[115102, 73743, 32, 131065, 62724, 5320, 65165, 71137] avg=65536.0000 max=131065 max_rank=3 min=32 min_rank=2 imbalance_ratio=1.999893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] avg=65536.0000 max=66193 max_rank=5 min=63258 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64872 max_load_reduction_pct=0.494961 moved_to_replicas=126279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] avg=65536.0000 max=66193 max_rank=5 min=63258 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64872 max_load_reduction_pct=0.494961 moved_to_replicas=126279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] avg=65536.0000 max=66193 max_rank=5 min=63258 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64872 max_load_reduction_pct=0.494961 moved_to_replicas=126279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] avg=65536.0000 max=66193 max_rank=5 min=63258 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64872 max_load_reduction_pct=0.494961 moved_to_replicas=126279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] avg=65536.0000 max=66193 max_rank=5 min=63258 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64872 max_load_reduction_pct=0.494961 moved_to_replicas=126279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[115102, 73743, 32, 131065, 62724, 5320, 65165, 71137] avg=65536.0000 max=131065 max_rank=3 min=32 min_rank=2 imbalance_ratio=1.999893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[115102, 73743, 32, 131065, 62724, 5320, 65165, 71137] avg=65536.0000 max=131065 max_rank=3 min=32 min_rank=2 imbalance_ratio=1.999893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[115102, 73743, 32, 131065, 62724, 5320, 65165, 71137] avg=65536.0000 max=131065 max_rank=3 min=32 min_rank=2 imbalance_ratio=1.999893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[47, 65669, 196117, 20, 64543, 47949, 3028, 146915] avg=65536.0000 max=196117 max_rank=2 min=20 min_rank=3 imbalance_ratio=2.992508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[47, 65669, 196117, 20, 64543, 47949, 3028, 146915] avg=65536.0000 max=196117 max_rank=2 min=20 min_rank=3 imbalance_ratio=2.992508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[47, 65669, 196117, 20, 64543, 47949, 3028, 146915] avg=65536.0000 max=196117 max_rank=2 min=20 min_rank=3 imbalance_ratio=2.992508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[47, 65669, 196117, 20, 64543, 47949, 3028, 146915] avg=65536.0000 max=196117 max_rank=2 min=20 min_rank=3 imbalance_ratio=2.992508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] avg=65536.0000 max=66193 max_rank=5 min=63258 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64872 max_load_reduction_pct=0.494961 moved_to_replicas=126279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] avg=65536.0000 max=66193 max_rank=5 min=63258 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64872 max_load_reduction_pct=0.494961 moved_to_replicas=126279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[66191, 66192, 64905, 66192, 63258, 66193, 65165, 66192] avg=65536.0000 max=66193 max_rank=5 min=63258 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64872 max_load_reduction_pct=0.494961 moved_to_replicas=126279 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=64443 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=129923 max_load_reduction_pct=0.662477 moved_to_replicas=210648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[47, 65669, 196117, 20, 64543, 47949, 3028, 146915] avg=65536.0000 max=196117 max_rank=2 min=20 min_rank=3 imbalance_ratio=2.992508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=64443 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=129923 max_load_reduction_pct=0.662477 moved_to_replicas=210648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=64443 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=129923 max_load_reduction_pct=0.662477 moved_to_replicas=210648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=64443 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=129923 max_load_reduction_pct=0.662477 moved_to_replicas=210648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=64443 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=129923 max_load_reduction_pct=0.662477 moved_to_replicas=210648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[47, 65669, 196117, 20, 64543, 47949, 3028, 146915] avg=65536.0000 max=196117 max_rank=2 min=20 min_rank=3 imbalance_ratio=2.992508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[47, 65669, 196117, 20, 64543, 47949, 3028, 146915] avg=65536.0000 max=196117 max_rank=2 min=20 min_rank=3 imbalance_ratio=2.992508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65359, 63527, 1483, 67165, 65505, 53882, 1081, 206286] avg=65536.0000 max=206286 max_rank=7 min=1081 min_rank=6 imbalance_ratio=3.147675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[47, 65669, 196117, 20, 64543, 47949, 3028, 146915] avg=65536.0000 max=196117 max_rank=2 min=20 min_rank=3 imbalance_ratio=2.992508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65359, 63527, 1483, 67165, 65505, 53882, 1081, 206286] avg=65536.0000 max=206286 max_rank=7 min=1081 min_rank=6 imbalance_ratio=3.147675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65359, 63527, 1483, 67165, 65505, 53882, 1081, 206286] avg=65536.0000 max=206286 max_rank=7 min=1081 min_rank=6 imbalance_ratio=3.147675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65359, 63527, 1483, 67165, 65505, 53882, 1081, 206286] avg=65536.0000 max=206286 max_rank=7 min=1081 min_rank=6 imbalance_ratio=3.147675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=64443 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=129923 max_load_reduction_pct=0.662477 moved_to_replicas=210648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=64443 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=129923 max_load_reduction_pct=0.662477 moved_to_replicas=210648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65359, 63527, 1483, 67165, 65505, 53882, 1081, 206286] avg=65536.0000 max=206286 max_rank=7 min=1081 min_rank=6 imbalance_ratio=3.147675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=64155 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=140093 max_load_reduction_pct=0.679120 moved_to_replicas=141067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[64443, 65669, 66192, 65549, 64543, 65506, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=64443 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=129923 max_load_reduction_pct=0.662477 moved_to_replicas=210648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=64155 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=140093 max_load_reduction_pct=0.679120 moved_to_replicas=141067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=64155 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=140093 max_load_reduction_pct=0.679120 moved_to_replicas=141067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=64155 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=140093 max_load_reduction_pct=0.679120 moved_to_replicas=141067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=64155 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=140093 max_load_reduction_pct=0.679120 moved_to_replicas=141067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65359, 63527, 1483, 67165, 65505, 53882, 1081, 206286] avg=65536.0000 max=206286 max_rank=7 min=1081 min_rank=6 imbalance_ratio=3.147675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65359, 63527, 1483, 67165, 65505, 53882, 1081, 206286] avg=65536.0000 max=206286 max_rank=7 min=1081 min_rank=6 imbalance_ratio=3.147675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[261336, 33482, 117621, 62897, 14733, 44, 44, 34131] avg=65536.0000 max=261336 max_rank=0 min=44 min_rank=5 imbalance_ratio=3.987671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[261336, 33482, 117621, 62897, 14733, 44, 44, 34131] avg=65536.0000 max=261336 max_rank=0 min=44 min_rank=5 imbalance_ratio=3.987671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65359, 63527, 1483, 67165, 65505, 53882, 1081, 206286] avg=65536.0000 max=206286 max_rank=7 min=1081 min_rank=6 imbalance_ratio=3.147675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[261336, 33482, 117621, 62897, 14733, 44, 44, 34131] avg=65536.0000 max=261336 max_rank=0 min=44 min_rank=5 imbalance_ratio=3.987671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[261336, 33482, 117621, 62897, 14733, 44, 44, 34131] avg=65536.0000 max=261336 max_rank=0 min=44 min_rank=5 imbalance_ratio=3.987671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=64155 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=140093 max_load_reduction_pct=0.679120 moved_to_replicas=141067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=64155 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=140093 max_load_reduction_pct=0.679120 moved_to_replicas=141067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[261336, 33482, 117621, 62897, 14733, 44, 44, 34131] avg=65536.0000 max=261336 max_rank=0 min=44 min_rank=5 imbalance_ratio=3.987671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] avg=65536.0000 max=66193 max_rank=2 min=62897 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=195143 max_load_reduction_pct=0.746713 moved_to_replicas=246573 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] avg=65536.0000 max=66193 max_rank=2 min=62897 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=195143 max_load_reduction_pct=0.746713 moved_to_replicas=246573 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65359, 64500, 66192, 66192, 65505, 64155, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=64155 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=140093 max_load_reduction_pct=0.679120 moved_to_replicas=141067 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] avg=65536.0000 max=66193 max_rank=2 min=62897 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=195143 max_load_reduction_pct=0.746713 moved_to_replicas=246573 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] avg=65536.0000 max=66193 max_rank=2 min=62897 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=195143 max_load_reduction_pct=0.746713 moved_to_replicas=246573 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] avg=65536.0000 max=66193 max_rank=2 min=62897 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=195143 max_load_reduction_pct=0.746713 moved_to_replicas=246573 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[261336, 33482, 117621, 62897, 14733, 44, 44, 34131] avg=65536.0000 max=261336 max_rank=0 min=44 min_rank=5 imbalance_ratio=3.987671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[261336, 33482, 117621, 62897, 14733, 44, 44, 34131] avg=65536.0000 max=261336 max_rank=0 min=44 min_rank=5 imbalance_ratio=3.987671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[68100, 17, 57414, 63524, 122844, 19, 146872, 65498] avg=65536.0000 max=146872 max_rank=6 min=17 min_rank=1 imbalance_ratio=2.241089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[68100, 17, 57414, 63524, 122844, 19, 146872, 65498] avg=65536.0000 max=146872 max_rank=6 min=17 min_rank=1 imbalance_ratio=2.241089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[261336, 33482, 117621, 62897, 14733, 44, 44, 34131] avg=65536.0000 max=261336 max_rank=0 min=44 min_rank=5 imbalance_ratio=3.987671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[68100, 17, 57414, 63524, 122844, 19, 146872, 65498] avg=65536.0000 max=146872 max_rank=6 min=17 min_rank=1 imbalance_ratio=2.241089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[68100, 17, 57414, 63524, 122844, 19, 146872, 65498] avg=65536.0000 max=146872 max_rank=6 min=17 min_rank=1 imbalance_ratio=2.241089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] avg=65536.0000 max=66193 max_rank=2 min=62897 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=195143 max_load_reduction_pct=0.746713 moved_to_replicas=246573 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] avg=65536.0000 max=66192 max_rank=0 min=63524 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80680 max_load_reduction_pct=0.549322 moved_to_replicas=139240 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] avg=65536.0000 max=66193 max_rank=2 min=62897 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=195143 max_load_reduction_pct=0.746713 moved_to_replicas=246573 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[68100, 17, 57414, 63524, 122844, 19, 146872, 65498] avg=65536.0000 max=146872 max_rank=6 min=17 min_rank=1 imbalance_ratio=2.241089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] avg=65536.0000 max=66192 max_rank=0 min=63524 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80680 max_load_reduction_pct=0.549322 moved_to_replicas=139240 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[66192, 65502, 66193, 62897, 66192, 65570, 65551, 66191] avg=65536.0000 max=66193 max_rank=2 min=62897 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=195143 max_load_reduction_pct=0.746713 moved_to_replicas=246573 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] avg=65536.0000 max=66192 max_rank=0 min=63524 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80680 max_load_reduction_pct=0.549322 moved_to_replicas=139240 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] avg=65536.0000 max=66192 max_rank=0 min=63524 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80680 max_load_reduction_pct=0.549322 moved_to_replicas=139240 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] avg=65536.0000 max=66192 max_rank=0 min=63524 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80680 max_load_reduction_pct=0.549322 moved_to_replicas=139240 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[68100, 17, 57414, 63524, 122844, 19, 146872, 65498] avg=65536.0000 max=146872 max_rank=6 min=17 min_rank=1 imbalance_ratio=2.241089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[66417, 139567, 130, 58286, 66210, 64168, 129286, 224] avg=65536.0000 max=139567 max_rank=1 min=130 min_rank=2 imbalance_ratio=2.129623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[68100, 17, 57414, 63524, 122844, 19, 146872, 65498] avg=65536.0000 max=146872 max_rank=6 min=17 min_rank=1 imbalance_ratio=2.241089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[66417, 139567, 130, 58286, 66210, 64168, 129286, 224] avg=65536.0000 max=139567 max_rank=1 min=130 min_rank=2 imbalance_ratio=2.129623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[68100, 17, 57414, 63524, 122844, 19, 146872, 65498] avg=65536.0000 max=146872 max_rank=6 min=17 min_rank=1 imbalance_ratio=2.241089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[66417, 139567, 130, 58286, 66210, 64168, 129286, 224] avg=65536.0000 max=139567 max_rank=1 min=130 min_rank=2 imbalance_ratio=2.129623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[66417, 139567, 130, 58286, 66210, 64168, 129286, 224] avg=65536.0000 max=139567 max_rank=1 min=130 min_rank=2 imbalance_ratio=2.129623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=73375 max_load_reduction_pct=0.525733 moved_to_replicas=136712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] avg=65536.0000 max=66192 max_rank=0 min=63524 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80680 max_load_reduction_pct=0.549322 moved_to_replicas=139240 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] avg=65536.0000 max=66192 max_rank=0 min=63524 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80680 max_load_reduction_pct=0.549322 moved_to_replicas=139240 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=73375 max_load_reduction_pct=0.525733 moved_to_replicas=136712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[66417, 139567, 130, 58286, 66210, 64168, 129286, 224] avg=65536.0000 max=139567 max_rank=1 min=130 min_rank=2 imbalance_ratio=2.129623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[66192, 65456, 65042, 63524, 66192, 66192, 66192, 65498] avg=65536.0000 max=66192 max_rank=0 min=63524 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80680 max_load_reduction_pct=0.549322 moved_to_replicas=139240 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=73375 max_load_reduction_pct=0.525733 moved_to_replicas=136712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=73375 max_load_reduction_pct=0.525733 moved_to_replicas=136712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=73375 max_load_reduction_pct=0.525733 moved_to_replicas=136712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[73219, 65183, 65500, 74974, 72709, 66899, 6, 105798] avg=65536.0000 max=105798 max_rank=7 min=6 min_rank=6 imbalance_ratio=1.614349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[66417, 139567, 130, 58286, 66210, 64168, 129286, 224] avg=65536.0000 max=139567 max_rank=1 min=130 min_rank=2 imbalance_ratio=2.129623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[73219, 65183, 65500, 74974, 72709, 66899, 6, 105798] avg=65536.0000 max=105798 max_rank=7 min=6 min_rank=6 imbalance_ratio=1.614349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[66417, 139567, 130, 58286, 66210, 64168, 129286, 224] avg=65536.0000 max=139567 max_rank=1 min=130 min_rank=2 imbalance_ratio=2.129623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[66417, 139567, 130, 58286, 66210, 64168, 129286, 224] avg=65536.0000 max=139567 max_rank=1 min=130 min_rank=2 imbalance_ratio=2.129623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[73219, 65183, 65500, 74974, 72709, 66899, 6, 105798] avg=65536.0000 max=105798 max_rank=7 min=6 min_rank=6 imbalance_ratio=1.614349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[73219, 65183, 65500, 74974, 72709, 66899, 6, 105798] avg=65536.0000 max=105798 max_rank=7 min=6 min_rank=6 imbalance_ratio=1.614349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] avg=65536.0000 max=66192 max_rank=0 min=62646 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39606 max_load_reduction_pct=0.374355 moved_to_replicas=62639 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=73375 max_load_reduction_pct=0.525733 moved_to_replicas=136712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] avg=65536.0000 max=66192 max_rank=0 min=62646 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39606 max_load_reduction_pct=0.374355 moved_to_replicas=62639 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=73375 max_load_reduction_pct=0.525733 moved_to_replicas=136712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[66192, 66192, 65658, 63502, 66192, 64168, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=73375 max_load_reduction_pct=0.525733 moved_to_replicas=136712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] avg=65536.0000 max=66192 max_rank=0 min=62646 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39606 max_load_reduction_pct=0.374355 moved_to_replicas=62639 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[73219, 65183, 65500, 74974, 72709, 66899, 6, 105798] avg=65536.0000 max=105798 max_rank=7 min=6 min_rank=6 imbalance_ratio=1.614349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] avg=65536.0000 max=66192 max_rank=0 min=62646 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39606 max_load_reduction_pct=0.374355 moved_to_replicas=62639 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65569, 6244, 128925, 30432, 93675, 66259, 2945, 130239] avg=65536.0000 max=130239 max_rank=7 min=2945 min_rank=6 imbalance_ratio=1.987289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65569, 6244, 128925, 30432, 93675, 66259, 2945, 130239] avg=65536.0000 max=130239 max_rank=7 min=2945 min_rank=6 imbalance_ratio=1.987289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] avg=65536.0000 max=66192 max_rank=0 min=62646 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39606 max_load_reduction_pct=0.374355 moved_to_replicas=62639 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[73219, 65183, 65500, 74974, 72709, 66899, 6, 105798] avg=65536.0000 max=105798 max_rank=7 min=6 min_rank=6 imbalance_ratio=1.614349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[73219, 65183, 65500, 74974, 72709, 66899, 6, 105798] avg=65536.0000 max=105798 max_rank=7 min=6 min_rank=6 imbalance_ratio=1.614349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[73219, 65183, 65500, 74974, 72709, 66899, 6, 105798] avg=65536.0000 max=105798 max_rank=7 min=6 min_rank=6 imbalance_ratio=1.614349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65569, 6244, 128925, 30432, 93675, 66259, 2945, 130239] avg=65536.0000 max=130239 max_rank=7 min=2945 min_rank=6 imbalance_ratio=1.987289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65569, 6244, 128925, 30432, 93675, 66259, 2945, 130239] avg=65536.0000 max=130239 max_rank=7 min=2945 min_rank=6 imbalance_ratio=1.987289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.491765 moved_to_replicas=154330 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.491765 moved_to_replicas=154330 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] avg=65536.0000 max=66192 max_rank=0 min=62646 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39606 max_load_reduction_pct=0.374355 moved_to_replicas=62639 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] avg=65536.0000 max=66192 max_rank=0 min=62646 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39606 max_load_reduction_pct=0.374355 moved_to_replicas=62639 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.491765 moved_to_replicas=154330 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65183, 65500, 66191, 66192, 66192, 62646, 66192] avg=65536.0000 max=66192 max_rank=0 min=62646 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39606 max_load_reduction_pct=0.374355 moved_to_replicas=62639 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.491765 moved_to_replicas=154330 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65569, 6244, 128925, 30432, 93675, 66259, 2945, 130239] avg=65536.0000 max=130239 max_rank=7 min=2945 min_rank=6 imbalance_ratio=1.987289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65721, 80917, 65486, 60955, 58778, 6, 61568, 130857] avg=65536.0000 max=130857 max_rank=7 min=6 min_rank=5 imbalance_ratio=1.996719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65721, 80917, 65486, 60955, 58778, 6, 61568, 130857] avg=65536.0000 max=130857 max_rank=7 min=6 min_rank=5 imbalance_ratio=1.996719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.491765 moved_to_replicas=154330 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65569, 6244, 128925, 30432, 93675, 66259, 2945, 130239] avg=65536.0000 max=130239 max_rank=7 min=2945 min_rank=6 imbalance_ratio=1.987289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65569, 6244, 128925, 30432, 93675, 66259, 2945, 130239] avg=65536.0000 max=130239 max_rank=7 min=2945 min_rank=6 imbalance_ratio=1.987289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65721, 80917, 65486, 60955, 58778, 6, 61568, 130857] avg=65536.0000 max=130857 max_rank=7 min=6 min_rank=5 imbalance_ratio=1.996719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65569, 6244, 128925, 30432, 93675, 66259, 2945, 130239] avg=65536.0000 max=130239 max_rank=7 min=2945 min_rank=6 imbalance_ratio=1.987289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65721, 80917, 65486, 60955, 58778, 6, 61568, 130857] avg=65536.0000 max=130857 max_rank=7 min=6 min_rank=5 imbalance_ratio=1.996719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] avg=65536.0000 max=66192 max_rank=1 min=63642 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64665 max_load_reduction_pct=0.494165 moved_to_replicas=79390 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] avg=65536.0000 max=66192 max_rank=1 min=63642 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64665 max_load_reduction_pct=0.494165 moved_to_replicas=79390 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.491765 moved_to_replicas=154330 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.491765 moved_to_replicas=154330 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] avg=65536.0000 max=66192 max_rank=1 min=63642 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64665 max_load_reduction_pct=0.494165 moved_to_replicas=79390 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65721, 80917, 65486, 60955, 58778, 6, 61568, 130857] avg=65536.0000 max=130857 max_rank=7 min=6 min_rank=5 imbalance_ratio=1.996719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] avg=65536.0000 max=66192 max_rank=1 min=63642 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64665 max_load_reduction_pct=0.494165 moved_to_replicas=79390 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65569, 66192, 66192, 61567, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61567 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.491765 moved_to_replicas=154330 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[6, 137149, 64376, 127768, 62933, 131043, 1002, 11] avg=65536.0000 max=137149 max_rank=1 min=6 min_rank=0 imbalance_ratio=2.092728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[6, 137149, 64376, 127768, 62933, 131043, 1002, 11] avg=65536.0000 max=137149 max_rank=1 min=6 min_rank=0 imbalance_ratio=2.092728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] avg=65536.0000 max=66192 max_rank=1 min=63642 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64665 max_load_reduction_pct=0.494165 moved_to_replicas=79390 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65721, 80917, 65486, 60955, 58778, 6, 61568, 130857] avg=65536.0000 max=130857 max_rank=7 min=6 min_rank=5 imbalance_ratio=1.996719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65721, 80917, 65486, 60955, 58778, 6, 61568, 130857] avg=65536.0000 max=130857 max_rank=7 min=6 min_rank=5 imbalance_ratio=1.996719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[6, 137149, 64376, 127768, 62933, 131043, 1002, 11] avg=65536.0000 max=137149 max_rank=1 min=6 min_rank=0 imbalance_ratio=2.092728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[6, 137149, 64376, 127768, 62933, 131043, 1002, 11] avg=65536.0000 max=137149 max_rank=1 min=6 min_rank=0 imbalance_ratio=2.092728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65721, 80917, 65486, 60955, 58778, 6, 61568, 130857] avg=65536.0000 max=130857 max_rank=7 min=6 min_rank=5 imbalance_ratio=1.996719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] avg=65536.0000 max=66192 max_rank=1 min=63757 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=70957 max_load_reduction_pct=0.517372 moved_to_replicas=197384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] avg=65536.0000 max=66192 max_rank=1 min=63757 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=70957 max_load_reduction_pct=0.517372 moved_to_replicas=197384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] avg=65536.0000 max=66192 max_rank=1 min=63642 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64665 max_load_reduction_pct=0.494165 moved_to_replicas=79390 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] avg=65536.0000 max=66192 max_rank=1 min=63642 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64665 max_load_reduction_pct=0.494165 moved_to_replicas=79390 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] avg=65536.0000 max=66192 max_rank=1 min=63757 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=70957 max_load_reduction_pct=0.517372 moved_to_replicas=197384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] avg=65536.0000 max=66192 max_rank=1 min=63757 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=70957 max_load_reduction_pct=0.517372 moved_to_replicas=197384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[6, 137149, 64376, 127768, 62933, 131043, 1002, 11] avg=65536.0000 max=137149 max_rank=1 min=6 min_rank=0 imbalance_ratio=2.092728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65721, 66192, 65486, 66192, 66192, 64671, 63642, 66192] avg=65536.0000 max=66192 max_rank=1 min=63642 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64665 max_load_reduction_pct=0.494165 moved_to_replicas=79390 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[130890, 122480, 63922, 10386, 65580, 65524, 65460, 46] avg=65536.0000 max=130890 max_rank=0 min=46 min_rank=7 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[130890, 122480, 63922, 10386, 65580, 65524, 65460, 46] avg=65536.0000 max=130890 max_rank=0 min=46 min_rank=7 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[6, 137149, 64376, 127768, 62933, 131043, 1002, 11] avg=65536.0000 max=137149 max_rank=1 min=6 min_rank=0 imbalance_ratio=2.092728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] avg=65536.0000 max=66192 max_rank=1 min=63757 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=70957 max_load_reduction_pct=0.517372 moved_to_replicas=197384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[6, 137149, 64376, 127768, 62933, 131043, 1002, 11] avg=65536.0000 max=137149 max_rank=1 min=6 min_rank=0 imbalance_ratio=2.092728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[130890, 122480, 63922, 10386, 65580, 65524, 65460, 46] avg=65536.0000 max=130890 max_rank=0 min=46 min_rank=7 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] avg=65536.0000 max=66194 max_rank=1 min=64404 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64696 max_load_reduction_pct=0.494278 moved_to_replicas=120986 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[130890, 122480, 63922, 10386, 65580, 65524, 65460, 46] avg=65536.0000 max=130890 max_rank=0 min=46 min_rank=7 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] avg=65536.0000 max=66194 max_rank=1 min=64404 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64696 max_load_reduction_pct=0.494278 moved_to_replicas=120986 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[6, 137149, 64376, 127768, 62933, 131043, 1002, 11] avg=65536.0000 max=137149 max_rank=1 min=6 min_rank=0 imbalance_ratio=2.092728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] avg=65536.0000 max=66192 max_rank=1 min=63757 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=70957 max_load_reduction_pct=0.517372 moved_to_replicas=197384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] avg=65536.0000 max=66192 max_rank=1 min=63757 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=70957 max_load_reduction_pct=0.517372 moved_to_replicas=197384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] avg=65536.0000 max=66194 max_rank=1 min=64404 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64696 max_load_reduction_pct=0.494278 moved_to_replicas=120986 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] avg=65536.0000 max=66194 max_rank=1 min=64404 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64696 max_load_reduction_pct=0.494278 moved_to_replicas=120986 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[130890, 122480, 63922, 10386, 65580, 65524, 65460, 46] avg=65536.0000 max=130890 max_rank=0 min=46 min_rank=7 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[65534, 66192, 64376, 66192, 63757, 66192, 65853, 66192] avg=65536.0000 max=66192 max_rank=1 min=63757 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=70957 max_load_reduction_pct=0.517372 moved_to_replicas=197384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] avg=65536.0000 max=66194 max_rank=1 min=64404 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64696 max_load_reduction_pct=0.494278 moved_to_replicas=120986 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[130890, 122480, 63922, 10386, 65580, 65524, 65460, 46] avg=65536.0000 max=130890 max_rank=0 min=46 min_rank=7 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[130890, 122480, 63922, 10386, 65580, 65524, 65460, 46] avg=65536.0000 max=130890 max_rank=0 min=46 min_rank=7 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[130890, 122480, 63922, 10386, 65580, 65524, 65460, 46] avg=65536.0000 max=130890 max_rank=0 min=46 min_rank=7 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] avg=65536.0000 max=66194 max_rank=1 min=64404 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64696 max_load_reduction_pct=0.494278 moved_to_replicas=120986 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] avg=65536.0000 max=66194 max_rank=1 min=64404 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64696 max_load_reduction_pct=0.494278 moved_to_replicas=120986 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[66192, 66194, 64404, 66190, 65580, 65524, 65460, 64744] avg=65536.0000 max=66194 max_rank=1 min=64404 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64696 max_load_reduction_pct=0.494278 moved_to_replicas=120986 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=12 total_expected_recv=786332 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190592 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=12 total_expected_recv=785470 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190976 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=12 total_expected_recv=789656 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187008 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=12 total_expected_recv=781730 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=195200 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=12 total_expected_recv=789737 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187264 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=12 total_expected_recv=792163 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=12 total_expected_recv=780381 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=196352 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=12 total_expected_recv=785987 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=190976 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 15:44:43] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 339.5 | throughput per GPU (TFLOP/s/GPU): 532.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.192005E+00 | loss scale: 1.0 | grad norm: 4.547 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +[after training is done] datetime: 2026-08-26 15:44:43 +[rank0]:[W826 15:44:44.458630081 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:44:45.943514738 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:44:45.992371056 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:44:45.003232013 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:44:45.003459027 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:44:45.003622094 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:44:45.003708206 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:44:46.492042643 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:44:52.388000 128245 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:44:52.388000 128245 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 128341 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -7314,4 +7303,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 44602 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 128245 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..2139cbd18b89ec76d4613afc4ab9d2e84fc28b95 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,208 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 165 +2, 629 +3, 629 +4, 165 +5, 629 +6, 629 +7, 629 +0, 1045 +1, 1043 +2, 1069 +3, 1065 +4, 1049 +5, 1057 +6, 1033 +7, 1031 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 2571 +1, 2571 +2, 2571 +3, 2571 +4, 2563 +5, 2549 +6, 2541 +7, 2569 +0, 43469 +1, 43469 +2, 22733 +3, 22733 +4, 43469 +5, 43469 +6, 43469 +7, 43469 +0, 52055 +1, 52055 +2, 52055 +3, 52055 +4, 51767 +5, 52055 +6, 52055 +7, 52055 +0, 53661 +1, 53661 +2, 53661 +3, 53661 +4, 53373 +5, 53661 +6, 53661 +7, 53661 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 55289 +1, 55289 +2, 55289 +3, 55289 +4, 55001 +5, 55289 +6, 55289 +7, 55289 +0, 57201 +1, 57201 +2, 57201 +3, 57201 +4, 56913 +5, 57201 +6, 57201 +7, 57201 +0, 68039 +1, 68195 +2, 68145 +3, 68675 +4, 68067 +5, 68183 +6, 68121 +7, 68183 +0, 90699 +1, 90755 +2, 90755 +3, 90755 +4, 90739 +5, 90755 +6, 90755 +7, 90755 +0, 91083 +1, 91139 +2, 91139 +3, 91139 +4, 91123 +5, 91139 +6, 91139 +7, 91139 +0, 91083 +1, 91139 +2, 91139 +3, 91139 +4, 91125 +5, 91139 +6, 91139 +7, 91139 +0, 91325 +1, 91379 +2, 91423 +3, 91459 +4, 91373 +5, 91437 +6, 91449 +7, 91371 +0, 95859 +1, 95925 +2, 95945 +3, 95891 +4, 96053 +5, 95901 +6, 95917 +7, 95927 +0, 108009 +1, 109161 +2, 109161 +3, 109161 +4, 109905 +5, 109161 +6, 109161 +7, 109161 +0, 122619 +1, 122739 +2, 122739 +3, 122739 +4, 122655 +5, 122739 +6, 122739 +7, 122739 +0, 122619 +1, 122737 +2, 122739 +3, 122739 +4, 122655 +5, 122739 +6, 122739 +7, 122739 +0, 3675 +1, 3678 +2, 3678 +3, 3678 +4, 3998 +5, 122737 +6, 5278 +7, 3678 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..7d6f4852e2eeb989c41f80772cb74dd8dc53194c --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3c4340d6a93e66a7433c21a7c2cde4febcdca41a --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,7305 @@ +[15:44:54] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:44:54] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:44:56.187000 131329 torch/distributed/run.py:766] +W0826 15:44:56.187000 131329 torch/distributed/run.py:766] ***************************************** +W0826 15:44:56.187000 131329 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:44:56.187000 131329 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 45 is connected to 6 is connected to 7 is connected to 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : +77 + +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0[Gloo] Rank +0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W826 15:45:14.374456916 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:45:14.374483502 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:45:14.375856494 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:45:14.380507113 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:45:14.380624909 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:45:14.380628714 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:45:14.380773026 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.032 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:45:14.549627281 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.893 seconds +time to initialize megatron (seconds): 7.073 +[after megatron is initialized] datetime: 2026-08-26 15:45:20 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8521882624 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:45:23 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:45:23 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3419.39, 3429.71) + train/valid/test-data-iterators-setup ..........: (71.13, 85.24) +[before the start of training step] datetime: 2026-08-26 15:45:23 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65455 expected_overflow=0 exact_recv_per_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65455 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65812 expected_overflow=0 exact_recv_per_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65812 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65754 expected_overflow=0 exact_recv_per_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65682 expected_overflow=0 exact_recv_per_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65363 expected_overflow=0 exact_recv_per_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65466 expected_overflow=0 exact_recv_per_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65754 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65562 expected_overflow=0 exact_recv_per_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65895 expected_overflow=0 exact_recv_per_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65562 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65363 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65682 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65466 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65895 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65383 expected_overflow=0 exact_recv_per_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65383 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65091 expected_overflow=0 exact_recv_per_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65091 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65828 expected_overflow=0 exact_recv_per_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65828 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65249 expected_overflow=0 exact_recv_per_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65580 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65545 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65249 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65357 expected_overflow=0 exact_recv_per_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65357 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65576 expected_overflow=0 exact_recv_per_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65576 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65403 expected_overflow=0 exact_recv_per_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65682 expected_overflow=0 exact_recv_per_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65806 expected_overflow=0 exact_recv_per_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65403 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65806 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65682 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65970 expected_overflow=0 exact_recv_per_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65300 expected_overflow=0 exact_recv_per_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65970 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65300 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65416 expected_overflow=0 exact_recv_per_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65416 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65640 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65311 expected_overflow=0 exact_recv_per_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66082 expected_overflow=0 exact_recv_per_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65311 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65979 expected_overflow=0 exact_recv_per_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=81920 expected_recv=66082 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65979 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65438 expected_overflow=0 exact_recv_per_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65438 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65581 expected_overflow=0 exact_recv_per_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65445 expected_overflow=0 exact_recv_per_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65144 expected_overflow=0 exact_recv_per_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65458 expected_overflow=0 exact_recv_per_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65581 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65603 expected_overflow=0 exact_recv_per_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65144 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65458 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65603 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65683 expected_overflow=0 exact_recv_per_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65615 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65445 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65683 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65442 expected_overflow=0 exact_recv_per_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65616 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65458 expected_overflow=0 exact_recv_per_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65545 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65442 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65458 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65228 expected_overflow=0 exact_recv_per_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65885 expected_overflow=0 exact_recv_per_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65885 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65491 expected_overflow=0 exact_recv_per_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65315 expected_overflow=0 exact_recv_per_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65491 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65626 expected_overflow=0 exact_recv_per_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65439 expected_overflow=0 exact_recv_per_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65315 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=81920 expected_recv=65228 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65768 expected_overflow=0 exact_recv_per_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65626 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65439 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65768 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65273 expected_overflow=0 exact_recv_per_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65759 expected_overflow=0 exact_recv_per_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65273 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65759 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65681 expected_overflow=0 exact_recv_per_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65249 expected_overflow=0 exact_recv_per_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66125 expected_overflow=0 exact_recv_per_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65681 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65475 expected_overflow=0 exact_recv_per_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65804 expected_overflow=0 exact_recv_per_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65249 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=81920 expected_recv=66125 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65475 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65804 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65416 expected_overflow=0 exact_recv_per_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65594 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65594 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=81920 expected_recv=65416 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65559 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65541 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65929 expected_overflow=0 exact_recv_per_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65776 expected_overflow=0 exact_recv_per_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65706 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65557 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65776 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65929 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65557 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65706 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66089 expected_overflow=0 exact_recv_per_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65489 expected_overflow=0 exact_recv_per_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65316 expected_overflow=0 exact_recv_per_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=81920 expected_recv=66089 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=81920 expected_recv=65489 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65316 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65709 expected_overflow=0 exact_recv_per_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65709 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65666 expected_overflow=0 exact_recv_per_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65147 expected_overflow=0 exact_recv_per_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65716 expected_overflow=0 exact_recv_per_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65666 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65147 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65716 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65767 expected_overflow=0 exact_recv_per_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65029 expected_overflow=0 exact_recv_per_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65767 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65117 expected_overflow=0 exact_recv_per_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65512 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=81920 expected_recv=65029 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65117 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65051 expected_overflow=0 exact_recv_per_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65445 expected_overflow=0 exact_recv_per_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65289 expected_overflow=0 exact_recv_per_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65051 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65445 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65289 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65721 expected_overflow=0 exact_recv_per_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65721 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65628 expected_overflow=0 exact_recv_per_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65696 expected_overflow=0 exact_recv_per_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65584 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=81920 expected_recv=65628 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65696 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65339 expected_overflow=0 exact_recv_per_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65332 expected_overflow=0 exact_recv_per_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65339 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65332 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65656 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65617 expected_overflow=0 exact_recv_per_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=81920 expected_recv=65617 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65342 expected_overflow=0 exact_recv_per_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65342 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65706 expected_overflow=0 exact_recv_per_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=81920 expected_recv=65706 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65453 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65453 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=81920 expected_recv=65496 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65708 expected_overflow=0 exact_recv_per_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=81920 expected_recv=65708 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65333 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65900 expected_overflow=0 exact_recv_per_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=81920 expected_recv=65333 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65900 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=81920 expected_recv=64977 expected_overflow=0 exact_recv_per_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65139 expected_overflow=0 exact_recv_per_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=81920 expected_recv=64977 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65139 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65133 expected_overflow=0 exact_recv_per_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=81920 expected_recv=65133 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65384 expected_overflow=0 exact_recv_per_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=81920 expected_recv=65384 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[65455, 65466, 65895, 65363, 65682, 65445, 65228, 65754] avg=65536.0000 max=65895 max_rank=2 min=65228 min_rank=6 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[65812, 65580, 65525, 65828, 65545, 65491, 65416, 65091] avg=65536.0000 max=65828 max_rank=3 min=65091 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[65562, 65403, 65682, 65576, 65970, 65249, 65489, 65357] avg=65536.0000 max=65970 max_rank=4 min=65249 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[65383, 65640, 65509, 65416, 66082, 65929, 65029, 65300] avg=65536.0000 max=66082 max_rank=4 min=65029 min_rank=6 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[65249, 65581, 65144, 65438, 65603, 65666, 65628, 65979] avg=65536.0000 max=65979 max_rank=7 min=65144 min_rank=2 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=12 total_expected_recv=787108 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183168 peak_demand=66125 required_capacity=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[65806, 65616, 65442, 65683, 65458, 65051, 65617, 65615] avg=65536.0000 max=65806 max_rank=0 min=65051 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[65311, 65315, 65439, 65885, 65768, 65339, 65706, 65525] avg=65536.0000 max=65885 max_rank=3 min=65311 min_rank=0 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=12 total_expected_recv=787065 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66089 required_capacity=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[65458, 65681, 65475, 65759, 65804, 65342, 65496, 65273] avg=65536.0000 max=65804 max_rank=4 min=65273 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=12 total_expected_recv=786919 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=65885 required_capacity=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=12 total_expected_recv=786345 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=65709 required_capacity=1.002640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=12 total_expected_recv=787393 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183040 peak_demand=66082 required_capacity=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=12 total_expected_recv=786478 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184448 peak_demand=65895 required_capacity=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[65545, 65541, 65706, 65559, 65557, 65453, 65333, 65594] avg=65536.0000 max=65706 max_rank=2 min=65333 min_rank=6 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[65626, 65709, 65716, 65316, 65147, 65708, 64977, 66089] avg=65536.0000 max=66089 max_rank=7 min=64977 min_rank=6 imbalance_ratio=1.008438 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[66125, 65117, 65289, 65512, 65445, 65900, 65133, 65767] avg=65536.0000 max=66125 max_rank=0 min=65117 min_rank=1 imbalance_ratio=1.008987 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[65776, 65696, 65656, 65584, 65332, 65139, 65384, 65721] avg=65536.0000 max=65776 max_rank=0 min=65139 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=12 total_expected_recv=785712 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184704 peak_demand=65929 required_capacity=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=12 total_expected_recv=784436 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187136 peak_demand=65706 required_capacity=1.002594 + [2026-08-26 15:45:50] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 26659.7 | throughput per GPU (TFLOP/s/GPU): 6.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115719E+01 | loss scale: 1.0 | grad norm: 44.805 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67200 | min safe capacity: 1.0254 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 73653.302734375 | max allocated: 80889.87255859375 | reserved: 86260.0 | max reserved: 86260.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65628 expected_overflow=0 exact_recv_per_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65308 expected_overflow=0 exact_recv_per_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65317 expected_overflow=0 exact_recv_per_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65946 expected_overflow=0 exact_recv_per_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65544 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65615 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65628 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65308 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65353 expected_overflow=0 exact_recv_per_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65317 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65946 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65577 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=81920 expected_recv=65353 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65600 expected_overflow=0 exact_recv_per_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65771 expected_overflow=0 exact_recv_per_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65600 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65771 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65567 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65698 expected_overflow=0 exact_recv_per_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65400 expected_overflow=0 exact_recv_per_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65698 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65105 expected_overflow=0 exact_recv_per_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65400 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65593 expected_overflow=0 exact_recv_per_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65105 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=81920 expected_recv=65593 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65872 expected_overflow=0 exact_recv_per_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65725 expected_overflow=0 exact_recv_per_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65459 expected_overflow=0 exact_recv_per_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65872 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65725 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65401 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65459 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65338 expected_overflow=0 exact_recv_per_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65504 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65463 expected_overflow=0 exact_recv_per_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65338 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65956 expected_overflow=0 exact_recv_per_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65166 expected_overflow=0 exact_recv_per_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65619 expected_overflow=0 exact_recv_per_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=81920 expected_recv=65463 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65956 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65166 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65619 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65584 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65894 expected_overflow=0 exact_recv_per_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65894 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65101 expected_overflow=0 exact_recv_per_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65561 expected_overflow=0 exact_recv_per_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65101 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65561 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65205 expected_overflow=0 exact_recv_per_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65429 expected_overflow=0 exact_recv_per_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65829 expected_overflow=0 exact_recv_per_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65205 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65429 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65829 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=81920 expected_recv=65407 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65881 expected_overflow=0 exact_recv_per_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65881 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65591 expected_overflow=0 exact_recv_per_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65793 expected_overflow=0 exact_recv_per_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65591 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65793 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65410 expected_overflow=0 exact_recv_per_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65240 expected_overflow=0 exact_recv_per_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65051 expected_overflow=0 exact_recv_per_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66041 expected_overflow=0 exact_recv_per_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65410 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65446 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65240 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=81920 expected_recv=65051 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=81920 expected_recv=66041 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65658 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65427 expected_overflow=0 exact_recv_per_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65355 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65427 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65269 expected_overflow=0 exact_recv_per_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65865 expected_overflow=0 exact_recv_per_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65711 expected_overflow=0 exact_recv_per_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65269 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65865 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=81920 expected_recv=65711 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65395 expected_overflow=0 exact_recv_per_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65604 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65395 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65733 expected_overflow=0 exact_recv_per_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65605 expected_overflow=0 exact_recv_per_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65733 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65859 expected_overflow=0 exact_recv_per_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65605 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65262 expected_overflow=0 exact_recv_per_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65673 expected_overflow=0 exact_recv_per_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65859 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=81920 expected_recv=65262 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65673 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65489 expected_overflow=0 exact_recv_per_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65489 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65994 expected_overflow=0 exact_recv_per_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65034 expected_overflow=0 exact_recv_per_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65994 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65385 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65676 expected_overflow=0 exact_recv_per_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65212 expected_overflow=0 exact_recv_per_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65034 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65676 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=81920 expected_recv=65212 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65361 expected_overflow=0 exact_recv_per_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65280 expected_overflow=0 exact_recv_per_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65361 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65255 expected_overflow=0 exact_recv_per_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65280 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65255 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65742 expected_overflow=0 exact_recv_per_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65914 expected_overflow=0 exact_recv_per_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65742 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65914 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66003 expected_overflow=0 exact_recv_per_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=81920 expected_recv=66003 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65414 expected_overflow=0 exact_recv_per_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65656 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=81920 expected_recv=65414 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65601 expected_overflow=0 exact_recv_per_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65492 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65117 expected_overflow=0 exact_recv_per_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65601 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65117 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65460 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65583 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65965 expected_overflow=0 exact_recv_per_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65543 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65965 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65745 expected_overflow=0 exact_recv_per_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65115 expected_overflow=0 exact_recv_per_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65745 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65433 expected_overflow=0 exact_recv_per_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65115 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65433 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65674 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65362 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65608 expected_overflow=0 exact_recv_per_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65406 expected_overflow=0 exact_recv_per_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65288 expected_overflow=0 exact_recv_per_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65608 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65406 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=81920 expected_recv=65288 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65363 expected_overflow=0 exact_recv_per_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65725 expected_overflow=0 exact_recv_per_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65363 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65725 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65695 expected_overflow=0 exact_recv_per_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65695 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65712 expected_overflow=0 exact_recv_per_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65712 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=81920 expected_recv=65496 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[65353, 65946, 65317, 65544, 65615, 65308, 65628, 65577] avg=65536.0000 max=65946 max_rank=1 min=65308 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[65593, 65698, 65400, 65600, 65771, 65567, 65554, 65105] avg=65536.0000 max=65771 max_rank=4 min=65105 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[65463, 65504, 65338, 65872, 65459, 65725, 65401, 65526] avg=65536.0000 max=65872 max_rank=3 min=65338 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[65407, 65894, 65101, 65956, 65584, 65166, 65619, 65561] avg=65536.0000 max=65956 max_rank=3 min=65101 min_rank=2 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[65051, 65881, 65793, 65205, 65509, 65429, 65829, 65591] avg=65536.0000 max=65881 max_rank=1 min=65051 min_rank=0 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[65711, 65658, 65427, 65446, 66041, 65410, 65240, 65355] avg=65536.0000 max=66041 max_rank=4 min=65240 min_rank=6 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[65262, 65395, 65605, 65865, 65604, 65269, 65555, 65733] avg=65536.0000 max=65865 max_rank=3 min=65262 min_rank=0 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[65212, 65489, 65034, 65859, 65534, 65493, 65673, 65994] avg=65536.0000 max=65994 max_rank=7 min=65034 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[65414, 65280, 66003, 65676, 65361, 65385, 65255, 65914] avg=65536.0000 max=66003 max_rank=2 min=65255 min_rank=6 imbalance_ratio=1.007126 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[65554, 65601, 65543, 65656, 65492, 65742, 65117, 65583] avg=65536.0000 max=65742 max_rank=5 min=65117 min_rank=6 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[65288, 65115, 65608, 65965, 65745, 65460, 65433, 65674] avg=65536.0000 max=65965 max_rank=3 min=65115 min_rank=1 imbalance_ratio=1.006546 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[65496, 65725, 65712, 65406, 65363, 65362, 65695, 65529] avg=65536.0000 max=65725 max_rank=1 min=65362 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=12 total_expected_recv=785316 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=65742 required_capacity=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=12 total_expected_recv=788050 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182528 peak_demand=65965 required_capacity=1.006546 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=12 total_expected_recv=787078 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183808 peak_demand=66041 required_capacity=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=12 total_expected_recv=787186 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183040 peak_demand=65946 required_capacity=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=12 total_expected_recv=785999 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=65829 required_capacity=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=12 total_expected_recv=787142 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184448 peak_demand=65994 required_capacity=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=12 total_expected_recv=784804 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185344 peak_demand=65711 required_capacity=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=12 total_expected_recv=785881 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=66003 required_capacity=1.007126 + [2026-08-26 15:45:51] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 919.3 | throughput per GPU (TFLOP/s/GPU): 196.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117269E+01 | loss scale: 1.0 | grad norm: 44.203 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67200 | min safe capacity: 1.0254 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65082 expected_overflow=0 exact_recv_per_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65320 expected_overflow=0 exact_recv_per_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65994 expected_overflow=0 exact_recv_per_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65939 expected_overflow=0 exact_recv_per_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65682 expected_overflow=0 exact_recv_per_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65082 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65320 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65559 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65994 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65939 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65682 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65174 expected_overflow=0 exact_recv_per_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=81920 expected_recv=65174 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65756 expected_overflow=0 exact_recv_per_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65295 expected_overflow=0 exact_recv_per_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65662 expected_overflow=0 exact_recv_per_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65851 expected_overflow=0 exact_recv_per_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65756 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65295 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65662 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65644 expected_overflow=0 exact_recv_per_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65681 expected_overflow=0 exact_recv_per_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65851 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65353 expected_overflow=0 exact_recv_per_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65644 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65681 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65353 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65046 expected_overflow=0 exact_recv_per_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65054 expected_overflow=0 exact_recv_per_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65618 expected_overflow=0 exact_recv_per_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=81920 expected_recv=65046 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65054 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65618 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65450 expected_overflow=0 exact_recv_per_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65383 expected_overflow=0 exact_recv_per_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65702 expected_overflow=0 exact_recv_per_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65450 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65751 expected_overflow=0 exact_recv_per_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65383 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65702 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65751 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65806 expected_overflow=0 exact_recv_per_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65661 expected_overflow=0 exact_recv_per_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65055 expected_overflow=0 exact_recv_per_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=81920 expected_recv=65806 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65661 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65055 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66063 expected_overflow=0 exact_recv_per_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65284 expected_overflow=0 exact_recv_per_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65104 expected_overflow=0 exact_recv_per_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65528 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=81920 expected_recv=66063 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65730 expected_overflow=0 exact_recv_per_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65284 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65104 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65730 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65863 expected_overflow=0 exact_recv_per_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65960 expected_overflow=0 exact_recv_per_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65439 expected_overflow=0 exact_recv_per_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=81920 expected_recv=65863 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65960 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65192 expected_overflow=0 exact_recv_per_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65439 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65898 expected_overflow=0 exact_recv_per_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65234 expected_overflow=0 exact_recv_per_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65898 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65520 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65234 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65822 expected_overflow=0 exact_recv_per_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65787 expected_overflow=0 exact_recv_per_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65223 expected_overflow=0 exact_recv_per_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65822 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65787 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=81920 expected_recv=65223 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65197 expected_overflow=0 exact_recv_per_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65517 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65224 expected_overflow=0 exact_recv_per_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65197 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65658 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65224 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65681 expected_overflow=0 exact_recv_per_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65381 expected_overflow=0 exact_recv_per_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65555 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65681 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=81920 expected_recv=65703 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65381 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65742 expected_overflow=0 exact_recv_per_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65279 expected_overflow=0 exact_recv_per_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65700 expected_overflow=0 exact_recv_per_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65742 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65512 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65279 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65700 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65137 expected_overflow=0 exact_recv_per_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65736 expected_overflow=0 exact_recv_per_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65383 expected_overflow=0 exact_recv_per_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65137 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65736 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=81920 expected_recv=65383 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65577 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65472 expected_overflow=0 exact_recv_per_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65781 expected_overflow=0 exact_recv_per_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65470 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65472 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65613 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65781 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=81920 expected_recv=64945 expected_overflow=0 exact_recv_per_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65644 expected_overflow=0 exact_recv_per_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=81920 expected_recv=64945 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65594 expected_overflow=0 exact_recv_per_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65644 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65489 expected_overflow=0 exact_recv_per_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=81920 expected_recv=65594 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65843 expected_overflow=0 exact_recv_per_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65964 expected_overflow=0 exact_recv_per_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65461 expected_overflow=0 exact_recv_per_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65489 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65843 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65602 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65964 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=81920 expected_recv=65461 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65983 expected_overflow=0 exact_recv_per_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=81920 expected_recv=64983 expected_overflow=0 exact_recv_per_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66001 expected_overflow=0 exact_recv_per_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65983 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=81920 expected_recv=64983 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65391 expected_overflow=0 exact_recv_per_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=81920 expected_recv=66001 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65349 expected_overflow=0 exact_recv_per_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65391 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65538 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65349 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65843 expected_overflow=0 exact_recv_per_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65106 expected_overflow=0 exact_recv_per_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65883 expected_overflow=0 exact_recv_per_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65843 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65106 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=81920 expected_recv=65883 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65618 expected_overflow=0 exact_recv_per_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65425 expected_overflow=0 exact_recv_per_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65460 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65618 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65425 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65762 expected_overflow=0 exact_recv_per_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65704 expected_overflow=0 exact_recv_per_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65762 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65176 expected_overflow=0 exact_recv_per_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65704 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=81920 expected_recv=65176 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65332 expected_overflow=0 exact_recv_per_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65756 expected_overflow=0 exact_recv_per_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65354 expected_overflow=0 exact_recv_per_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65583 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65332 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65756 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65354 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65438 expected_overflow=0 exact_recv_per_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65438 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=81920 expected_recv=65552 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[65174, 65538, 65559, 65082, 65320, 65939, 65682, 65994] avg=65536.0000 max=65994 max_rank=7 min=65082 min_rank=3 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[65046, 65756, 65851, 65295, 65662, 65644, 65353, 65681] avg=65536.0000 max=65851 max_rank=2 min=65046 min_rank=0 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[65806, 65524, 65450, 65054, 65618, 65383, 65751, 65702] avg=65536.0000 max=65806 max_rank=0 min=65054 min_rank=3 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[65863, 65055, 66063, 65661, 65528, 65284, 65730, 65104] avg=65536.0000 max=66063 max_rank=2 min=65055 min_rank=1 imbalance_ratio=1.008041 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[65223, 65439, 65898, 65960, 65192, 65234, 65822, 65520] avg=65536.0000 max=65960 max_rank=3 min=65192 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[65703, 65517, 65658, 65787, 65197, 65224, 65681, 65521] avg=65536.0000 max=65787 max_rank=3 min=65197 min_rank=4 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[65383, 65381, 65512, 65555, 65742, 65700, 65736, 65279] avg=65536.0000 max=65742 max_rank=4 min=65279 min_rank=7 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[65594, 65577, 65472, 65137, 65470, 65781, 65644, 65613] avg=65536.0000 max=65781 max_rank=5 min=65137 min_rank=3 imbalance_ratio=1.003738 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[66001, 65489, 65602, 64945, 65843, 65964, 64983, 65461] avg=65536.0000 max=66001 max_rank=0 min=64945 min_rank=3 imbalance_ratio=1.007095 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[65883, 65391, 65509, 65983, 65538, 65349, 65106, 65529] avg=65536.0000 max=65983 max_rank=3 min=65106 min_rank=6 imbalance_ratio=1.006821 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65176, 65460, 65618, 65843, 65425, 65530, 65704, 65532] avg=65536.0000 max=65843 max_rank=3 min=65176 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[65552, 65511, 65583, 65762, 65756, 65332, 65438, 65354] avg=65536.0000 max=65762 max_rank=3 min=65332 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=12 total_expected_recv=786064 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185344 peak_demand=65983 required_capacity=1.006821 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=12 total_expected_recv=786364 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=65964 required_capacity=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=12 total_expected_recv=786290 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=65994 required_capacity=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=12 total_expected_recv=785638 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185088 peak_demand=65756 required_capacity=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=12 total_expected_recv=787775 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66063 required_capacity=1.008041 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=12 total_expected_recv=786404 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184192 peak_demand=66001 required_capacity=1.007095 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=12 total_expected_recv=786630 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184192 peak_demand=65822 required_capacity=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=12 total_expected_recv=786291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=65843 required_capacity=1.004684 + [2026-08-26 15:45:51] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 320.5 | throughput per GPU (TFLOP/s/GPU): 564.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.699974E+00 | loss scale: 1.0 | grad norm: 69.455 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65591 expected_overflow=0 exact_recv_per_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65240 expected_overflow=0 exact_recv_per_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65634 expected_overflow=0 exact_recv_per_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65444 expected_overflow=0 exact_recv_per_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65807 expected_overflow=0 exact_recv_per_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65834 expected_overflow=0 exact_recv_per_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65193 expected_overflow=0 exact_recv_per_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65591 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65240 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65634 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65444 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65807 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65834 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65193 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=81920 expected_recv=65545 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65471 expected_overflow=0 exact_recv_per_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65637 expected_overflow=0 exact_recv_per_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65471 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65689 expected_overflow=0 exact_recv_per_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65784 expected_overflow=0 exact_recv_per_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65637 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65366 expected_overflow=0 exact_recv_per_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65689 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65354 expected_overflow=0 exact_recv_per_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65784 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65424 expected_overflow=0 exact_recv_per_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65366 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65354 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65563 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=81920 expected_recv=65424 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65826 expected_overflow=0 exact_recv_per_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65796 expected_overflow=0 exact_recv_per_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65532 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65826 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65665 expected_overflow=0 exact_recv_per_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64814 expected_overflow=0 exact_recv_per_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65821 expected_overflow=0 exact_recv_per_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65796 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65321 expected_overflow=0 exact_recv_per_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65665 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=81920 expected_recv=64814 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65821 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=81920 expected_recv=65321 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65056 expected_overflow=0 exact_recv_per_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65056 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65765 expected_overflow=0 exact_recv_per_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65380 expected_overflow=0 exact_recv_per_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65579 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65340 expected_overflow=0 exact_recv_per_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65721 expected_overflow=0 exact_recv_per_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65765 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65797 expected_overflow=0 exact_recv_per_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65380 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65340 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65721 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65797 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65650 expected_overflow=0 exact_recv_per_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65595 expected_overflow=0 exact_recv_per_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=81920 expected_recv=65650 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65595 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65389 expected_overflow=0 exact_recv_per_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65389 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65717 expected_overflow=0 exact_recv_per_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65641 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65717 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65454 expected_overflow=0 exact_recv_per_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65449 expected_overflow=0 exact_recv_per_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65454 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65528 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65449 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65400 expected_overflow=0 exact_recv_per_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65400 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65620 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65607 expected_overflow=0 exact_recv_per_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65734 expected_overflow=0 exact_recv_per_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65607 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65734 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65720 expected_overflow=0 exact_recv_per_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65476 expected_overflow=0 exact_recv_per_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65212 expected_overflow=0 exact_recv_per_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65720 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65519 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65367 expected_overflow=0 exact_recv_per_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65476 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=81920 expected_recv=65212 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65367 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65996 expected_overflow=0 exact_recv_per_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65996 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65374 expected_overflow=0 exact_recv_per_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65151 expected_overflow=0 exact_recv_per_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65374 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65151 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65747 expected_overflow=0 exact_recv_per_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65399 expected_overflow=0 exact_recv_per_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65527 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65727 expected_overflow=0 exact_recv_per_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66121 expected_overflow=0 exact_recv_per_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65747 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65399 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=81920 expected_recv=65727 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=81920 expected_recv=66121 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65377 expected_overflow=0 exact_recv_per_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65377 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65140 expected_overflow=0 exact_recv_per_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65140 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65546 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65616 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65455 expected_overflow=0 exact_recv_per_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66184 expected_overflow=0 exact_recv_per_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65503 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65455 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=81920 expected_recv=66184 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65679 expected_overflow=0 exact_recv_per_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65679 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65582 expected_overflow=0 exact_recv_per_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65582 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65565 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65615 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65035 expected_overflow=0 exact_recv_per_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65374 expected_overflow=0 exact_recv_per_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65254 expected_overflow=0 exact_recv_per_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65442 expected_overflow=0 exact_recv_per_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65007 expected_overflow=0 exact_recv_per_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65035 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65374 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=81920 expected_recv=65254 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65442 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65007 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65964 expected_overflow=0 exact_recv_per_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65668 expected_overflow=0 exact_recv_per_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65964 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65668 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65249 expected_overflow=0 exact_recv_per_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65479 expected_overflow=0 exact_recv_per_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65249 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65782 expected_overflow=0 exact_recv_per_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65697 expected_overflow=0 exact_recv_per_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65276 expected_overflow=0 exact_recv_per_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65479 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65782 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=81920 expected_recv=65697 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65276 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65499 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65294 expected_overflow=0 exact_recv_per_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65974 expected_overflow=0 exact_recv_per_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65294 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65974 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65947 expected_overflow=0 exact_recv_per_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66052 expected_overflow=0 exact_recv_per_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65947 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65119 expected_overflow=0 exact_recv_per_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65717 expected_overflow=0 exact_recv_per_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65127 expected_overflow=0 exact_recv_per_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=81920 expected_recv=66052 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65119 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65629 expected_overflow=0 exact_recv_per_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65717 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=81920 expected_recv=65127 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65629 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65214 expected_overflow=0 exact_recv_per_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65332 expected_overflow=0 exact_recv_per_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65214 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65332 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65731 expected_overflow=0 exact_recv_per_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65473 expected_overflow=0 exact_recv_per_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65638 expected_overflow=0 exact_recv_per_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65731 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65473 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=81920 expected_recv=65638 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[65545, 65591, 65240, 65634, 65807, 65193, 65444, 65834] avg=65536.0000 max=65834 max_rank=7 min=65193 min_rank=5 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65424, 65471, 65637, 65689, 65784, 65563, 65366, 65354] avg=65536.0000 max=65784 max_rank=4 min=65354 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[65321, 65513, 65532, 65826, 65796, 64814, 65665, 65821] avg=65536.0000 max=65826 max_rank=3 min=64814 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65650, 65056, 65765, 65579, 65380, 65721, 65340, 65797] avg=65536.0000 max=65797 max_rank=7 min=65056 min_rank=1 imbalance_ratio=1.003983 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[65515, 65595, 65641, 65389, 65717, 65528, 65454, 65449] avg=65536.0000 max=65717 max_rank=4 min=65389 min_rank=3 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[65212, 65400, 65607, 65620, 65734, 65720, 65519, 65476] avg=65536.0000 max=65734 max_rank=4 min=65212 min_rank=0 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[65727, 65367, 65374, 65996, 65151, 65527, 65747, 65399] avg=65536.0000 max=65996 max_rank=3 min=65151 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[65530, 66121, 65140, 65377, 65546, 65616, 65503, 65455] avg=65536.0000 max=66121 max_rank=1 min=65140 min_rank=2 imbalance_ratio=1.008926 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[65035, 66184, 65582, 65679, 65565, 65615, 65374, 65254] avg=65536.0000 max=66184 max_rank=1 min=65035 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[65479, 65442, 65964, 65007, 65668, 65249, 65782, 65697] avg=65536.0000 max=65964 max_rank=2 min=65007 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[66052, 65276, 65294, 65499, 65974, 65947, 65119, 65127] avg=65536.0000 max=66052 max_rank=0 min=65119 min_rank=6 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[65731, 65717, 65214, 65629, 65332, 65554, 65473, 65638] avg=65536.0000 max=65731 max_rank=0 min=65214 min_rank=2 imbalance_ratio=1.002975 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=12 total_expected_recv=786733 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183296 peak_demand=66184 required_capacity=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=12 total_expected_recv=786924 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183680 peak_demand=65996 required_capacity=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=12 total_expected_recv=786047 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=65947 required_capacity=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=12 total_expected_recv=787454 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182656 peak_demand=65974 required_capacity=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=12 total_expected_recv=785990 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=65964 required_capacity=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=12 total_expected_recv=786221 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=66052 required_capacity=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=12 total_expected_recv=786301 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=65834 required_capacity=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=12 total_expected_recv=785786 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=65782 required_capacity=1.003754 + [2026-08-26 15:45:52] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 319.3 | throughput per GPU (TFLOP/s/GPU): 566.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.340186E+01 | loss scale: 1.0 | grad norm: 19.515 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65776 expected_overflow=0 exact_recv_per_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65618 expected_overflow=0 exact_recv_per_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65467 expected_overflow=0 exact_recv_per_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65817 expected_overflow=0 exact_recv_per_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65776 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65133 expected_overflow=0 exact_recv_per_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65618 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65502 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65467 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65386 expected_overflow=0 exact_recv_per_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65817 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65589 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65133 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=81920 expected_recv=65386 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65245 expected_overflow=0 exact_recv_per_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65448 expected_overflow=0 exact_recv_per_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65369 expected_overflow=0 exact_recv_per_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65245 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65448 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65420 expected_overflow=0 exact_recv_per_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65369 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65575 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66012 expected_overflow=0 exact_recv_per_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65420 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65656 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=81920 expected_recv=65563 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=81920 expected_recv=66012 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65461 expected_overflow=0 exact_recv_per_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65461 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65662 expected_overflow=0 exact_recv_per_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65711 expected_overflow=0 exact_recv_per_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65408 expected_overflow=0 exact_recv_per_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65393 expected_overflow=0 exact_recv_per_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65695 expected_overflow=0 exact_recv_per_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65527 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65431 expected_overflow=0 exact_recv_per_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65662 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65711 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65408 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65393 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65695 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=81920 expected_recv=65431 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65421 expected_overflow=0 exact_recv_per_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65421 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65897 expected_overflow=0 exact_recv_per_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65218 expected_overflow=0 exact_recv_per_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65848 expected_overflow=0 exact_recv_per_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65218 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65897 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65287 expected_overflow=0 exact_recv_per_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65651 expected_overflow=0 exact_recv_per_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65845 expected_overflow=0 exact_recv_per_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65121 expected_overflow=0 exact_recv_per_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65848 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65287 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65651 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65845 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=81920 expected_recv=65121 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65465 expected_overflow=0 exact_recv_per_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65324 expected_overflow=0 exact_recv_per_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65465 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65324 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66011 expected_overflow=0 exact_recv_per_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65266 expected_overflow=0 exact_recv_per_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65590 expected_overflow=0 exact_recv_per_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65679 expected_overflow=0 exact_recv_per_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65452 expected_overflow=0 exact_recv_per_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=81920 expected_recv=66011 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65266 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65925 expected_overflow=0 exact_recv_per_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65590 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65679 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=81920 expected_recv=65452 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65925 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66089 expected_overflow=0 exact_recv_per_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66089 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64980 expected_overflow=0 exact_recv_per_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65214 expected_overflow=0 exact_recv_per_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65224 expected_overflow=0 exact_recv_per_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65081 expected_overflow=0 exact_recv_per_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=81920 expected_recv=64980 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65214 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65583 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65709 expected_overflow=0 exact_recv_per_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65224 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=81920 expected_recv=65081 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65709 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65906 expected_overflow=0 exact_recv_per_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65285 expected_overflow=0 exact_recv_per_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65906 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65285 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65216 expected_overflow=0 exact_recv_per_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65860 expected_overflow=0 exact_recv_per_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65199 expected_overflow=0 exact_recv_per_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65216 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65604 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65860 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65199 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65355 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65567 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65496 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65629 expected_overflow=0 exact_recv_per_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65665 expected_overflow=0 exact_recv_per_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65693 expected_overflow=0 exact_recv_per_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65629 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65665 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65693 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65362 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65720 expected_overflow=0 exact_recv_per_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65984 expected_overflow=0 exact_recv_per_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65720 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65489 expected_overflow=0 exact_recv_per_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65984 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65489 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64939 expected_overflow=0 exact_recv_per_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65567 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65518 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65812 expected_overflow=0 exact_recv_per_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65684 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65387 expected_overflow=0 exact_recv_per_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=81920 expected_recv=64939 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65812 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=81920 expected_recv=65387 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65261 expected_overflow=0 exact_recv_per_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65622 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65261 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65476 expected_overflow=0 exact_recv_per_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65655 expected_overflow=0 exact_recv_per_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65476 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65570 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65860 expected_overflow=0 exact_recv_per_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65655 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65454 expected_overflow=0 exact_recv_per_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65394 expected_overflow=0 exact_recv_per_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65498 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65860 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65454 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=81920 expected_recv=65394 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65602 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65607 expected_overflow=0 exact_recv_per_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65244 expected_overflow=0 exact_recv_per_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65590 expected_overflow=0 exact_recv_per_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65607 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65244 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65287 expected_overflow=0 exact_recv_per_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65590 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65769 expected_overflow=0 exact_recv_per_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65287 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65644 expected_overflow=0 exact_recv_per_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65769 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65774 expected_overflow=0 exact_recv_per_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65569 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=81920 expected_recv=65644 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65774 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65490 expected_overflow=0 exact_recv_per_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65490 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65361 expected_overflow=0 exact_recv_per_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65864 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65385 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65361 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65076 expected_overflow=0 exact_recv_per_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=81920 expected_recv=65076 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[65386, 65502, 65618, 65133, 65776, 65589, 65817, 65467] avg=65536.0000 max=65817 max_rank=6 min=65133 min_rank=3 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[66012, 65369, 65448, 65563, 65245, 65656, 65420, 65575] avg=65536.0000 max=66012 max_rank=0 min=65245 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65431, 65662, 65527, 65695, 65461, 65393, 65408, 65711] avg=65536.0000 max=65711 max_rank=7 min=65393 min_rank=5 imbalance_ratio=1.002670 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[65651, 65218, 65897, 65121, 65421, 65845, 65287, 65848] avg=65536.0000 max=65897 max_rank=2 min=65121 min_rank=3 imbalance_ratio=1.005508 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66221, 65081, 65925, 65224, 65214, 64951] avg=65536.0000 max=66221 max_rank=2 min=64951 min_rank=7 imbalance_ratio=1.010452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66221, 65081, 65925, 65224, 65214, 64951] avg=65536.0000 max=66221 max_rank=2 min=64951 min_rank=7 imbalance_ratio=1.010452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66221, 65081, 65925, 65224, 65214, 64951] avg=65536.0000 max=66221 max_rank=2 min=64951 min_rank=7 imbalance_ratio=1.010452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] avg=65536.0000 max=66192 max_rank=2 min=64980 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29 max_load_reduction_pct=0.000438 moved_to_replicas=29 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[65590, 65465, 65324, 65452, 65501, 65679, 65266, 66011] avg=65536.0000 max=66011 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.007248 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] avg=65536.0000 max=66192 max_rank=2 min=64980 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29 max_load_reduction_pct=0.000438 moved_to_replicas=29 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] avg=65536.0000 max=66192 max_rank=2 min=64980 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29 max_load_reduction_pct=0.000438 moved_to_replicas=29 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66221, 65081, 65925, 65224, 65214, 64951] avg=65536.0000 max=66221 max_rank=2 min=64951 min_rank=7 imbalance_ratio=1.010452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66221, 65081, 65925, 65224, 65214, 64951] avg=65536.0000 max=66221 max_rank=2 min=64951 min_rank=7 imbalance_ratio=1.010452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66221, 65081, 65925, 65224, 65214, 64951] avg=65536.0000 max=66221 max_rank=2 min=64951 min_rank=7 imbalance_ratio=1.010452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66221, 65081, 65925, 65224, 65214, 64951] avg=65536.0000 max=66221 max_rank=2 min=64951 min_rank=7 imbalance_ratio=1.010452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] avg=65536.0000 max=66192 max_rank=2 min=64980 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29 max_load_reduction_pct=0.000438 moved_to_replicas=29 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66221, 65081, 65925, 65224, 65214, 64951] avg=65536.0000 max=66221 max_rank=2 min=64951 min_rank=7 imbalance_ratio=1.010452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] avg=65536.0000 max=66192 max_rank=2 min=64980 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29 max_load_reduction_pct=0.000438 moved_to_replicas=29 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] avg=65536.0000 max=66192 max_rank=2 min=64980 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29 max_load_reduction_pct=0.000438 moved_to_replicas=29 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] avg=65536.0000 max=66192 max_rank=2 min=64980 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29 max_load_reduction_pct=0.000438 moved_to_replicas=29 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[65583, 66089, 66192, 65081, 65925, 65224, 65214, 64980] avg=65536.0000 max=66192 max_rank=2 min=64980 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29 max_load_reduction_pct=0.000438 moved_to_replicas=29 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65860, 65906, 65285, 65509, 65709, 65199, 65604, 65216] avg=65536.0000 max=65906 max_rank=1 min=65199 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[65693, 65567, 65496, 65521, 65355, 65362, 65665, 65629] avg=65536.0000 max=65693 max_rank=0 min=65355 min_rank=4 imbalance_ratio=1.002396 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[65684, 65984, 65489, 65387, 65720, 64939, 65518, 65567] avg=65536.0000 max=65984 max_rank=1 min=64939 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65655, 65622, 65261, 65394, 65812, 65498, 65570, 65476] avg=65536.0000 max=65812 max_rank=4 min=65261 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65590, 65454, 65602, 65644, 65860, 65287, 65244, 65607] avg=65536.0000 max=65860 max_rank=4 min=65244 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[65385, 65569, 65774, 65076, 65769, 65361, 65864, 65490] avg=65536.0000 max=65864 max_rank=6 min=65076 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=12 total_expected_recv=787407 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66089 required_capacity=1.008438 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=12 total_expected_recv=786913 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=12 total_expected_recv=787554 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184192 peak_demand=65925 required_capacity=1.005936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=12 total_expected_recv=785032 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185088 peak_demand=65845 required_capacity=1.004715 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=12 total_expected_recv=786577 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=66011 required_capacity=1.007248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=12 total_expected_recv=785877 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185216 peak_demand=65864 required_capacity=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=12 total_expected_recv=784576 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186880 peak_demand=65695 required_capacity=1.002426 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=12 total_expected_recv=787520 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66012 required_capacity=1.007263 + [2026-08-26 15:45:52] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 318.8 | throughput per GPU (TFLOP/s/GPU): 567.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.040801E+01 | loss scale: 1.0 | grad norm: 88.846 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65344 expected_overflow=0 exact_recv_per_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65378 expected_overflow=0 exact_recv_per_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65802 expected_overflow=0 exact_recv_per_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65903 expected_overflow=0 exact_recv_per_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=81920 expected_recv=64901 expected_overflow=0 exact_recv_per_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65444 expected_overflow=0 exact_recv_per_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65344 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65604 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65378 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65912 expected_overflow=0 exact_recv_per_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65802 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65903 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=81920 expected_recv=64901 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65444 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=81920 expected_recv=65912 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65720 expected_overflow=0 exact_recv_per_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65378 expected_overflow=0 exact_recv_per_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65409 expected_overflow=0 exact_recv_per_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65887 expected_overflow=0 exact_recv_per_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65384 expected_overflow=0 exact_recv_per_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65211 expected_overflow=0 exact_recv_per_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65720 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65378 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65439 expected_overflow=0 exact_recv_per_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65409 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65887 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65384 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65860 expected_overflow=0 exact_recv_per_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65211 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65439 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=81920 expected_recv=65860 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65507 expected_overflow=0 exact_recv_per_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66144 expected_overflow=0 exact_recv_per_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65172 expected_overflow=0 exact_recv_per_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65417 expected_overflow=0 exact_recv_per_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65507 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65827 expected_overflow=0 exact_recv_per_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65172 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=81920 expected_recv=66144 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65584 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65417 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65827 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65143 expected_overflow=0 exact_recv_per_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65494 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=81920 expected_recv=65143 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65415 expected_overflow=0 exact_recv_per_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65062 expected_overflow=0 exact_recv_per_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65415 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65776 expected_overflow=0 exact_recv_per_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65513 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65842 expected_overflow=0 exact_recv_per_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65929 expected_overflow=0 exact_recv_per_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65062 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65776 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65842 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65423 expected_overflow=0 exact_recv_per_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65929 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65328 expected_overflow=0 exact_recv_per_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65423 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=81920 expected_recv=65328 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65233 expected_overflow=0 exact_recv_per_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65722 expected_overflow=0 exact_recv_per_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65706 expected_overflow=0 exact_recv_per_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65233 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65722 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65706 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65601 expected_overflow=0 exact_recv_per_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65567 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65458 expected_overflow=0 exact_recv_per_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65470 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65601 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=81920 expected_recv=65458 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65777 expected_overflow=0 exact_recv_per_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65777 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65243 expected_overflow=0 exact_recv_per_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65759 expected_overflow=0 exact_recv_per_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65226 expected_overflow=0 exact_recv_per_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65857 expected_overflow=0 exact_recv_per_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65243 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65759 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65933 expected_overflow=0 exact_recv_per_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65226 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65305 expected_overflow=0 exact_recv_per_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65857 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65188 expected_overflow=0 exact_recv_per_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65933 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65305 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=81920 expected_recv=65188 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65060 expected_overflow=0 exact_recv_per_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65723 expected_overflow=0 exact_recv_per_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65060 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65543 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65723 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65786 expected_overflow=0 exact_recv_per_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65487 expected_overflow=0 exact_recv_per_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65609 expected_overflow=0 exact_recv_per_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65786 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65487 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=81920 expected_recv=65609 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65164 expected_overflow=0 exact_recv_per_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65164 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65437 expected_overflow=0 exact_recv_per_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65589 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65979 expected_overflow=0 exact_recv_per_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65437 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65374 expected_overflow=0 exact_recv_per_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65979 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65578 expected_overflow=0 exact_recv_per_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65374 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65392 expected_overflow=0 exact_recv_per_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65775 expected_overflow=0 exact_recv_per_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65578 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65369 expected_overflow=0 exact_recv_per_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65392 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=81920 expected_recv=65775 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65369 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65384 expected_overflow=0 exact_recv_per_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65384 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65750 expected_overflow=0 exact_recv_per_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65270 expected_overflow=0 exact_recv_per_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65695 expected_overflow=0 exact_recv_per_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65750 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65776 expected_overflow=0 exact_recv_per_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65270 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65695 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65803 expected_overflow=0 exact_recv_per_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65241 expected_overflow=0 exact_recv_per_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65776 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65803 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=81920 expected_recv=65241 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65375 expected_overflow=0 exact_recv_per_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65477 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65375 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65683 expected_overflow=0 exact_recv_per_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65586 expected_overflow=0 exact_recv_per_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65785 expected_overflow=0 exact_recv_per_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65683 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65828 expected_overflow=0 exact_recv_per_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65586 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65785 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=81920 expected_recv=64954 expected_overflow=0 exact_recv_per_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65828 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65600 expected_overflow=0 exact_recv_per_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=81920 expected_recv=64954 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65857 expected_overflow=0 exact_recv_per_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65833 expected_overflow=0 exact_recv_per_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=81920 expected_recv=65600 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65857 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65833 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65792 expected_overflow=0 exact_recv_per_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65340 expected_overflow=0 exact_recv_per_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65338 expected_overflow=0 exact_recv_per_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65792 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65340 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65633 expected_overflow=0 exact_recv_per_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65338 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65633 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65002 expected_overflow=0 exact_recv_per_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65777 expected_overflow=0 exact_recv_per_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65389 expected_overflow=0 exact_recv_per_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=81920 expected_recv=65002 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65777 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65389 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65698 expected_overflow=0 exact_recv_per_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65425 expected_overflow=0 exact_recv_per_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65698 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65412 expected_overflow=0 exact_recv_per_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65425 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65618 expected_overflow=0 exact_recv_per_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65412 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65176 expected_overflow=0 exact_recv_per_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65793 expected_overflow=0 exact_recv_per_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65618 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65176 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=81920 expected_recv=65793 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[65912, 65344, 65802, 65903, 65604, 64901, 65378, 65444] avg=65536.0000 max=65912 max_rank=0 min=64901 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[65860, 65720, 65887, 65211, 65409, 65384, 65378, 65439] avg=65536.0000 max=65887 max_rank=2 min=65211 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[65143, 65507, 65417, 65827, 65172, 65584, 66144, 65494] avg=65536.0000 max=66144 max_rank=6 min=65143 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[65328, 65415, 65842, 65929, 65513, 65776, 65062, 65423] avg=65536.0000 max=65929 max_rank=3 min=65062 min_rank=6 imbalance_ratio=1.005997 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65458, 65531, 65567, 65470, 65233, 65706, 65722, 65601] avg=65536.0000 max=65722 max_rank=6 min=65233 min_rank=4 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[65188, 65777, 65857, 65933, 65759, 65243, 65226, 65305] avg=65536.0000 max=65933 max_rank=3 min=65188 min_rank=0 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[65609, 65549, 65723, 65786, 65060, 65531, 65543, 65487] avg=65536.0000 max=65786 max_rank=3 min=65060 min_rank=4 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65775, 65164, 65374, 65578, 65437, 65589, 65979, 65392] avg=65536.0000 max=65979 max_rank=6 min=65164 min_rank=1 imbalance_ratio=1.006760 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[65241, 65369, 65695, 65776, 65270, 65384, 65750, 65803] avg=65536.0000 max=65803 max_rank=7 min=65241 min_rank=0 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65600, 65477, 65785, 65828, 65586, 65375, 65683, 64954] avg=65536.0000 max=65828 max_rank=3 min=64954 min_rank=7 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[65002, 65857, 65338, 65633, 65340, 65833, 65792, 65493] avg=65536.0000 max=65857 max_rank=1 min=65002 min_rank=0 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65793, 65777, 65412, 65618, 65425, 65389, 65698, 65176] avg=65536.0000 max=65793 max_rank=0 min=65176 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=12 total_expected_recv=785695 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185088 peak_demand=65833 required_capacity=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=12 total_expected_recv=786487 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184064 peak_demand=65857 required_capacity=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=12 total_expected_recv=787355 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183168 peak_demand=66144 required_capacity=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=12 total_expected_recv=788492 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183040 peak_demand=65933 required_capacity=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=12 total_expected_recv=784808 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186240 peak_demand=65759 required_capacity=1.003403 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=12 total_expected_recv=785909 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185088 peak_demand=65912 required_capacity=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=12 total_expected_recv=787699 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182528 peak_demand=65887 required_capacity=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=12 total_expected_recv=785011 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185728 peak_demand=65803 required_capacity=1.004074 + [2026-08-26 15:45:52] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 317.7 | throughput per GPU (TFLOP/s/GPU): 569.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.012807E+01 | loss scale: 1.0 | grad norm: 26.748 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65475 expected_overflow=0 exact_recv_per_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65455 expected_overflow=0 exact_recv_per_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65449 expected_overflow=0 exact_recv_per_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65341 expected_overflow=0 exact_recv_per_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65114 expected_overflow=0 exact_recv_per_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65475 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65455 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65753 expected_overflow=0 exact_recv_per_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65449 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65341 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65114 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=81920 expected_recv=65753 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65331 expected_overflow=0 exact_recv_per_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65319 expected_overflow=0 exact_recv_per_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65331 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65803 expected_overflow=0 exact_recv_per_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65622 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65718 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65319 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65803 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65767 expected_overflow=0 exact_recv_per_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65469 expected_overflow=0 exact_recv_per_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65218 expected_overflow=0 exact_recv_per_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65791 expected_overflow=0 exact_recv_per_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65700 expected_overflow=0 exact_recv_per_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65767 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65218 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65791 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=81920 expected_recv=65469 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65700 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65656 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65482 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65448 expected_overflow=0 exact_recv_per_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65448 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65394 expected_overflow=0 exact_recv_per_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65563 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65394 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65631 expected_overflow=0 exact_recv_per_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65686 expected_overflow=0 exact_recv_per_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65226 expected_overflow=0 exact_recv_per_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65471 expected_overflow=0 exact_recv_per_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65631 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65686 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=81920 expected_recv=65226 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65471 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65176 expected_overflow=0 exact_recv_per_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65542 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65176 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65790 expected_overflow=0 exact_recv_per_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65795 expected_overflow=0 exact_recv_per_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65790 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65795 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65198 expected_overflow=0 exact_recv_per_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65768 expected_overflow=0 exact_recv_per_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65198 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65825 expected_overflow=0 exact_recv_per_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65111 expected_overflow=0 exact_recv_per_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66035 expected_overflow=0 exact_recv_per_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65768 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66041 expected_overflow=0 exact_recv_per_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=81920 expected_recv=65825 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65111 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=81920 expected_recv=66035 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65446 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=81920 expected_recv=66041 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65276 expected_overflow=0 exact_recv_per_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65276 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65486 expected_overflow=0 exact_recv_per_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65415 expected_overflow=0 exact_recv_per_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65486 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65145 expected_overflow=0 exact_recv_per_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65701 expected_overflow=0 exact_recv_per_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65415 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65706 expected_overflow=0 exact_recv_per_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=81920 expected_recv=65145 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65385 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65701 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65706 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=81920 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65259 expected_overflow=0 exact_recv_per_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65259 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65295 expected_overflow=0 exact_recv_per_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65371 expected_overflow=0 exact_recv_per_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65295 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65278 expected_overflow=0 exact_recv_per_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65371 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65410 expected_overflow=0 exact_recv_per_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65746 expected_overflow=0 exact_recv_per_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65755 expected_overflow=0 exact_recv_per_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65286 expected_overflow=0 exact_recv_per_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=81920 expected_recv=65278 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65410 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65755 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65746 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65286 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65500 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66042 expected_overflow=0 exact_recv_per_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65462 expected_overflow=0 exact_recv_per_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=81920 expected_recv=66042 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65260 expected_overflow=0 exact_recv_per_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65451 expected_overflow=0 exact_recv_per_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65418 expected_overflow=0 exact_recv_per_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65475 expected_overflow=0 exact_recv_per_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65462 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65810 expected_overflow=0 exact_recv_per_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=81920 expected_recv=65260 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65451 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65418 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65475 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65810 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65460 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65650 expected_overflow=0 exact_recv_per_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65650 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65902 expected_overflow=0 exact_recv_per_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65026 expected_overflow=0 exact_recv_per_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65683 expected_overflow=0 exact_recv_per_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65902 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65153 expected_overflow=0 exact_recv_per_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65692 expected_overflow=0 exact_recv_per_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=81920 expected_recv=65654 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65026 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65683 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65153 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65438 expected_overflow=0 exact_recv_per_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65692 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65438 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65641 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=81920 expected_recv=65604 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65461 expected_overflow=0 exact_recv_per_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65229 expected_overflow=0 exact_recv_per_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65667 expected_overflow=0 exact_recv_per_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65461 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65229 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65667 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65628 expected_overflow=0 exact_recv_per_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65628 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65972 expected_overflow=0 exact_recv_per_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65463 expected_overflow=0 exact_recv_per_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65972 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=81920 expected_recv=65463 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65652 expected_overflow=0 exact_recv_per_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65350 expected_overflow=0 exact_recv_per_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65249 expected_overflow=0 exact_recv_per_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65652 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65350 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65933 expected_overflow=0 exact_recv_per_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65249 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65933 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65009 expected_overflow=0 exact_recv_per_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65267 expected_overflow=0 exact_recv_per_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65009 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=81920 expected_recv=65267 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65798 expected_overflow=0 exact_recv_per_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65710 expected_overflow=0 exact_recv_per_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65254 expected_overflow=0 exact_recv_per_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[66202, 65753, 65104, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66202 max_rank=0 min=65104 min_rank=2 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65798 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65710 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65254 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[66202, 65753, 65104, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66202 max_rank=0 min=65104 min_rank=2 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[66202, 65753, 65104, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66202 max_rank=0 min=65104 min_rank=2 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65801 expected_overflow=0 exact_recv_per_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[66202, 65753, 65104, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66202 max_rank=0 min=65104 min_rank=2 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66192 max_rank=0 min=65114 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000151 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=81920 expected_recv=65801 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66192 max_rank=0 min=65114 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000151 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66192 max_rank=0 min=65114 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000151 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66192 max_rank=0 min=65114 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000151 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[66202, 65753, 65104, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66202 max_rank=0 min=65104 min_rank=2 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[66202, 65753, 65104, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66202 max_rank=0 min=65104 min_rank=2 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[66202, 65753, 65104, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66202 max_rank=0 min=65104 min_rank=2 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[66202, 65753, 65104, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66202 max_rank=0 min=65104 min_rank=2 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66192 max_rank=0 min=65114 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000151 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66192 max_rank=0 min=65114 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000151 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66192 max_rank=0 min=65114 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000151 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[66192, 65753, 65114, 65509, 65475, 65455, 65341, 65449] avg=65536.0000 max=66192 max_rank=0 min=65114 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000151 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65469, 65501, 65803, 65718, 65622, 65331, 65525, 65319] avg=65536.0000 max=65803 max_rank=2 min=65319 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65226, 65448, 65482, 65791, 65218, 65767, 65656, 65700] avg=65536.0000 max=65791 max_rank=3 min=65218 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65825, 65176, 65542, 65631, 65394, 65563, 65471, 65686] avg=65536.0000 max=65825 max_rank=0 min=65176 min_rank=1 imbalance_ratio=1.004410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[65145, 65446, 66035, 65198, 65795, 65790, 65111, 65768] avg=65536.0000 max=66035 max_rank=2 min=65111 min_rank=6 imbalance_ratio=1.007614 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65252, 66199, 65746, 65371] avg=65536.0000 max=66199 max_rank=5 min=65252 min_rank=4 imbalance_ratio=1.010117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65252, 66199, 65746, 65371] avg=65536.0000 max=66199 max_rank=5 min=65252 min_rank=4 imbalance_ratio=1.010117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65252, 66199, 65746, 65371] avg=65536.0000 max=66199 max_rank=5 min=65252 min_rank=4 imbalance_ratio=1.010117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] avg=65536.0000 max=66192 max_rank=5 min=65259 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000106 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65252, 66199, 65746, 65371] avg=65536.0000 max=66199 max_rank=5 min=65252 min_rank=4 imbalance_ratio=1.010117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] avg=65536.0000 max=66192 max_rank=5 min=65259 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000106 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] avg=65536.0000 max=66192 max_rank=5 min=65259 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000106 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[65278, 65706, 65701, 65486, 65276, 66041, 65385, 65415] avg=65536.0000 max=66041 max_rank=5 min=65276 min_rank=4 imbalance_ratio=1.007706 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] avg=65536.0000 max=66192 max_rank=5 min=65259 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000106 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65252, 66199, 65746, 65371] avg=65536.0000 max=66199 max_rank=5 min=65252 min_rank=4 imbalance_ratio=1.010117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65252, 66199, 65746, 65371] avg=65536.0000 max=66199 max_rank=5 min=65252 min_rank=4 imbalance_ratio=1.010117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65252, 66199, 65746, 65371] avg=65536.0000 max=66199 max_rank=5 min=65252 min_rank=4 imbalance_ratio=1.010117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65252, 66199, 65746, 65371] avg=65536.0000 max=66199 max_rank=5 min=65252 min_rank=4 imbalance_ratio=1.010117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] avg=65536.0000 max=66192 max_rank=5 min=65259 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000106 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] avg=65536.0000 max=66192 max_rank=5 min=65259 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000106 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] avg=65536.0000 max=66192 max_rank=5 min=65259 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000106 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65260, 65755, 65410, 65295, 65259, 66192, 65746, 65371] avg=65536.0000 max=66192 max_rank=5 min=65259 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000106 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[65654, 65475, 65451, 66042, 65500, 65286, 65418, 65462] avg=65536.0000 max=66042 max_rank=3 min=65286 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[65604, 65153, 65026, 65650, 65460, 65810, 65683, 65902] avg=65536.0000 max=65902 max_rank=7 min=65026 min_rank=2 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[65463, 65523, 65461, 65540, 65438, 65692, 65530, 65641] avg=65536.0000 max=65692 max_rank=5 min=65438 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[65267, 65523, 65350, 65628, 65667, 65229, 65652, 65972] avg=65536.0000 max=65972 max_rank=7 min=65229 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[65801, 65254, 65710, 65933, 65249, 65534, 65798, 65009] avg=65536.0000 max=65933 max_rank=3 min=65009 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=12 total_expected_recv=787690 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184064 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=12 total_expected_recv=787421 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66042 required_capacity=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=12 total_expected_recv=785353 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185600 peak_demand=65795 required_capacity=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=12 total_expected_recv=786694 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=65972 required_capacity=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=12 total_expected_recv=786316 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185600 peak_demand=65798 required_capacity=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=12 total_expected_recv=786085 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=66035 required_capacity=1.007614 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=12 total_expected_recv=785713 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=65755 required_capacity=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=12 total_expected_recv=786184 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=66192 required_capacity=1.010010 + [2026-08-26 15:45:53] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 317.9 | throughput per GPU (TFLOP/s/GPU): 568.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.024440E+01 | loss scale: 1.0 | grad norm: 6.230 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65587 expected_overflow=0 exact_recv_per_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65903 expected_overflow=0 exact_recv_per_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65259 expected_overflow=0 exact_recv_per_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65366 expected_overflow=0 exact_recv_per_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65264 expected_overflow=0 exact_recv_per_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66040 expected_overflow=0 exact_recv_per_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65903 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65587 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65377 expected_overflow=0 exact_recv_per_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65492 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65259 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65366 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65264 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=81920 expected_recv=66040 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=81920 expected_recv=65377 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65752 expected_overflow=0 exact_recv_per_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65662 expected_overflow=0 exact_recv_per_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65831 expected_overflow=0 exact_recv_per_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65752 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65662 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65831 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65323 expected_overflow=0 exact_recv_per_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65394 expected_overflow=0 exact_recv_per_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65109 expected_overflow=0 exact_recv_per_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65323 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65674 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65394 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65109 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=81920 expected_recv=65543 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65954 expected_overflow=0 exact_recv_per_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65824 expected_overflow=0 exact_recv_per_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65414 expected_overflow=0 exact_recv_per_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65954 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65824 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65414 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65454 expected_overflow=0 exact_recv_per_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65208 expected_overflow=0 exact_recv_per_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65308 expected_overflow=0 exact_recv_per_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65454 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65208 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65568 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65308 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65397 expected_overflow=0 exact_recv_per_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65414 expected_overflow=0 exact_recv_per_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65649 expected_overflow=0 exact_recv_per_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65397 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65414 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65649 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65875 expected_overflow=0 exact_recv_per_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65557 expected_overflow=0 exact_recv_per_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65363 expected_overflow=0 exact_recv_per_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65491 expected_overflow=0 exact_recv_per_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65542 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65875 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65557 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65363 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=81920 expected_recv=65491 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66119 expected_overflow=0 exact_recv_per_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65217 expected_overflow=0 exact_recv_per_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65232 expected_overflow=0 exact_recv_per_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=81920 expected_recv=66119 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65217 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65232 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65265 expected_overflow=0 exact_recv_per_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65732 expected_overflow=0 exact_recv_per_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65265 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65495 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65732 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65531 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65697 expected_overflow=0 exact_recv_per_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=81920 expected_recv=65697 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65450 expected_overflow=0 exact_recv_per_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65463 expected_overflow=0 exact_recv_per_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65510 expected_overflow=0 exact_recv_per_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65450 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65463 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65510 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65351 expected_overflow=0 exact_recv_per_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65467 expected_overflow=0 exact_recv_per_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65911 expected_overflow=0 exact_recv_per_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65495 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65911 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65351 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65467 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65921 expected_overflow=0 exact_recv_per_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=81920 expected_recv=65641 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65866 expected_overflow=0 exact_recv_per_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65242 expected_overflow=0 exact_recv_per_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65921 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65866 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65242 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=81920 expected_recv=64926 expected_overflow=0 exact_recv_per_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65752 expected_overflow=0 exact_recv_per_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65448 expected_overflow=0 exact_recv_per_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=81920 expected_recv=64926 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65477 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65448 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65752 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65345 expected_overflow=0 exact_recv_per_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65863 expected_overflow=0 exact_recv_per_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=81920 expected_recv=65656 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65345 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65863 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65460 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65269 expected_overflow=0 exact_recv_per_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65435 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65269 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65835 expected_overflow=0 exact_recv_per_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65545 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65633 expected_overflow=0 exact_recv_per_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=81920 expected_recv=65835 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65633 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65703 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65487 expected_overflow=0 exact_recv_per_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65681 expected_overflow=0 exact_recv_per_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65487 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65681 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65452 expected_overflow=0 exact_recv_per_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65288 expected_overflow=0 exact_recv_per_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65452 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65288 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65547 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65435 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65814 expected_overflow=0 exact_recv_per_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65504 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65814 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65642 expected_overflow=0 exact_recv_per_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65505 expected_overflow=0 exact_recv_per_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65287 expected_overflow=0 exact_recv_per_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65903 expected_overflow=0 exact_recv_per_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65642 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65505 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65775 expected_overflow=0 exact_recv_per_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=81920 expected_recv=65287 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65903 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65494 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65775 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65236 expected_overflow=0 exact_recv_per_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65236 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65333 expected_overflow=0 exact_recv_per_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65330 expected_overflow=0 exact_recv_per_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65724 expected_overflow=0 exact_recv_per_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65333 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65849 expected_overflow=0 exact_recv_per_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65817 expected_overflow=0 exact_recv_per_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65330 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=81920 expected_recv=65724 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66042 expected_overflow=0 exact_recv_per_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65849 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65817 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=81920 expected_recv=66042 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65576 expected_overflow=0 exact_recv_per_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65528 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65576 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64993 expected_overflow=0 exact_recv_per_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65425 expected_overflow=0 exact_recv_per_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=81920 expected_recv=64993 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65058 expected_overflow=0 exact_recv_per_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65425 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=81920 expected_recv=65058 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[65377, 66040, 65492, 65903, 65587, 65259, 65366, 65264] avg=65536.0000 max=66040 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65543, 65109, 65831, 65662, 65752, 65323, 65674, 65394] avg=65536.0000 max=65831 max_rank=2 min=65109 min_rank=1 imbalance_ratio=1.004501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[65558, 65308, 65414, 65954, 65824, 65454, 65208, 65568] avg=65536.0000 max=65954 max_rank=3 min=65208 min_rank=6 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[65491, 65363, 65414, 65397, 65649, 65875, 65557, 65542] avg=65536.0000 max=65875 max_rank=5 min=65363 min_rank=1 imbalance_ratio=1.005173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[65697, 65531, 66119, 65217, 65232, 65495, 65265, 65732] avg=65536.0000 max=66119 max_rank=2 min=65217 min_rank=3 imbalance_ratio=1.008896 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65641, 65911, 65450, 65463, 65510, 65467, 65495, 65351] avg=65536.0000 max=65911 max_rank=1 min=65351 min_rank=7 imbalance_ratio=1.005722 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65656, 65477, 65921, 65866, 65242, 65752, 64926, 65448] avg=65536.0000 max=65921 max_rank=2 min=64926 min_rank=6 imbalance_ratio=1.005875 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[65835, 65269, 65345, 65863, 65460, 65536, 65435, 65545] avg=65536.0000 max=65863 max_rank=3 min=65269 min_rank=1 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65509, 65681, 65535, 65633, 65703, 65288, 65487, 65452] avg=65536.0000 max=65703 max_rank=4 min=65288 min_rank=5 imbalance_ratio=1.002548 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65287, 65814, 65547, 65554, 65435, 65505, 65504, 65642] avg=65536.0000 max=65814 max_rank=1 min=65287 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65724, 65493, 65903, 65494, 65775, 65330, 65236, 65333] avg=65536.0000 max=65903 max_rank=2 min=65236 min_rank=6 imbalance_ratio=1.005600 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[65425, 65576, 65849, 65817, 66042, 65058, 65528, 64993] avg=65536.0000 max=66042 max_rank=4 min=64993 min_rank=7 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=12 total_expected_recv=787823 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182528 peak_demand=65954 required_capacity=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=12 total_expected_recv=787820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=182656 peak_demand=66119 required_capacity=1.008896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=12 total_expected_recv=784681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=65674 required_capacity=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=12 total_expected_recv=787211 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=66042 required_capacity=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=12 total_expected_recv=786572 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184448 peak_demand=66040 required_capacity=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=12 total_expected_recv=785342 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185344 peak_demand=65875 required_capacity=1.005173 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=12 total_expected_recv=785264 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=187136 peak_demand=65732 required_capacity=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=12 total_expected_recv=786743 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184064 peak_demand=65835 required_capacity=1.004562 + [2026-08-26 15:45:53] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 317.9 | throughput per GPU (TFLOP/s/GPU): 568.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.794078E+00 | loss scale: 1.0 | grad norm: 5.600 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65110 expected_overflow=0 exact_recv_per_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65476 expected_overflow=0 exact_recv_per_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65808 expected_overflow=0 exact_recv_per_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65428 expected_overflow=0 exact_recv_per_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65110 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65476 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65620 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65426 expected_overflow=0 exact_recv_per_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65808 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65428 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65864 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65556 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=81920 expected_recv=65426 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65805 expected_overflow=0 exact_recv_per_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65191 expected_overflow=0 exact_recv_per_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65640 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65805 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65627 expected_overflow=0 exact_recv_per_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65461 expected_overflow=0 exact_recv_per_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65191 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65516 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65627 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65461 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=81920 expected_recv=65494 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65691 expected_overflow=0 exact_recv_per_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65272 expected_overflow=0 exact_recv_per_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65343 expected_overflow=0 exact_recv_per_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65717 expected_overflow=0 exact_recv_per_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65691 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65452 expected_overflow=0 exact_recv_per_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65272 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65343 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65699 expected_overflow=0 exact_recv_per_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65717 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65452 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65699 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=81920 expected_recv=65621 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65784 expected_overflow=0 exact_recv_per_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65857 expected_overflow=0 exact_recv_per_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65316 expected_overflow=0 exact_recv_per_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65576 expected_overflow=0 exact_recv_per_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65784 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65857 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65428 expected_overflow=0 exact_recv_per_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65576 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65316 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65237 expected_overflow=0 exact_recv_per_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65523 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65428 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65237 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=81920 expected_recv=65567 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65660 expected_overflow=0 exact_recv_per_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65758 expected_overflow=0 exact_recv_per_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65660 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65477 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65537 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65758 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65412 expected_overflow=0 exact_recv_per_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65521 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65412 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65358 expected_overflow=0 exact_recv_per_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65565 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=81920 expected_recv=65358 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65258 expected_overflow=0 exact_recv_per_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65547 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65545 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65258 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65935 expected_overflow=0 exact_recv_per_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65074 expected_overflow=0 exact_recv_per_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65928 expected_overflow=0 exact_recv_per_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65935 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65399 expected_overflow=0 exact_recv_per_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65602 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65074 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65928 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=81920 expected_recv=65399 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65424 expected_overflow=0 exact_recv_per_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65306 expected_overflow=0 exact_recv_per_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65424 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65306 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65763 expected_overflow=0 exact_recv_per_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65214 expected_overflow=0 exact_recv_per_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65371 expected_overflow=0 exact_recv_per_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65775 expected_overflow=0 exact_recv_per_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65763 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65736 expected_overflow=0 exact_recv_per_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65214 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65371 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65699 expected_overflow=0 exact_recv_per_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65775 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65736 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=81920 expected_recv=65699 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64768 expected_overflow=0 exact_recv_per_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65255 expected_overflow=0 exact_recv_per_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=81920 expected_recv=64768 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65255 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65897 expected_overflow=0 exact_recv_per_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65359 expected_overflow=0 exact_recv_per_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66155 expected_overflow=0 exact_recv_per_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65622 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65897 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65359 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=81920 expected_recv=66155 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65825 expected_overflow=0 exact_recv_per_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65407 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=81920 expected_recv=65825 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65269 expected_overflow=0 exact_recv_per_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65437 expected_overflow=0 exact_recv_per_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65269 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65848 expected_overflow=0 exact_recv_per_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65437 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65474 expected_overflow=0 exact_recv_per_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65265 expected_overflow=0 exact_recv_per_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65848 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65419 expected_overflow=0 exact_recv_per_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65474 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65790 expected_overflow=0 exact_recv_per_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65786 expected_overflow=0 exact_recv_per_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65265 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65419 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65790 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=81920 expected_recv=65786 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65614 expected_overflow=0 exact_recv_per_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65867 expected_overflow=0 exact_recv_per_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65614 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65867 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65669 expected_overflow=0 exact_recv_per_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65669 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65178 expected_overflow=0 exact_recv_per_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65224 expected_overflow=0 exact_recv_per_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65613 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65178 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65630 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=81920 expected_recv=65224 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65321 expected_overflow=0 exact_recv_per_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65321 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65730 expected_overflow=0 exact_recv_per_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65730 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65243 expected_overflow=0 exact_recv_per_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65243 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65573 expected_overflow=0 exact_recv_per_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65406 expected_overflow=0 exact_recv_per_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65658 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65841 expected_overflow=0 exact_recv_per_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65573 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65406 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65516 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=81920 expected_recv=65841 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65672 expected_overflow=0 exact_recv_per_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65672 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65665 expected_overflow=0 exact_recv_per_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65665 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65460 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65229 expected_overflow=0 exact_recv_per_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65500 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65433 expected_overflow=0 exact_recv_per_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65775 expected_overflow=0 exact_recv_per_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65229 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65433 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=81920 expected_recv=65775 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[65426, 65110, 65808, 65864, 65476, 65620, 65556, 65428] avg=65536.0000 max=65864 max_rank=3 min=65110 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65494, 65640, 65516, 65627, 65554, 65805, 65461, 65191] avg=65536.0000 max=65805 max_rank=5 min=65191 min_rank=7 imbalance_ratio=1.004105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[65621, 65493, 65717, 65452, 65691, 65272, 65699, 65343] avg=65536.0000 max=65717 max_rank=2 min=65272 min_rank=5 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65567, 65784, 65523, 65428, 65857, 65316, 65237, 65576] avg=65536.0000 max=65857 max_rank=4 min=65237 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[65358, 65660, 65521, 65565, 65477, 65758, 65412, 65537] avg=65536.0000 max=65758 max_rank=5 min=65358 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65399, 65547, 65074, 65928, 65258, 65935, 65602, 65545] avg=65536.0000 max=65935 max_rank=5 min=65074 min_rank=2 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65699, 65424, 65371, 65736, 65763, 65214, 65775, 65306] avg=65536.0000 max=65775 max_rank=6 min=65214 min_rank=5 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[65825, 64768, 65359, 65407, 65622, 65897, 66155, 65255] avg=65536.0000 max=66155 max_rank=6 min=64768 min_rank=1 imbalance_ratio=1.009445 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65786, 65269, 65265, 65790, 65474, 65848, 65419, 65437] avg=65536.0000 max=65848 max_rank=5 min=65265 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65630, 65614, 65178, 65224, 65493, 65669, 65613, 65867] avg=65536.0000 max=65867 max_rank=7 min=65178 min_rank=2 imbalance_ratio=1.005051 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[65516, 65321, 65406, 65841, 65658, 65243, 65573, 65730] avg=65536.0000 max=65841 max_rank=3 min=65243 min_rank=5 imbalance_ratio=1.004654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[65775, 65672, 65554, 65433, 65500, 65460, 65229, 65665] avg=65536.0000 max=65775 max_rank=0 min=65229 min_rank=6 imbalance_ratio=1.003647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=12 total_expected_recv=787037 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184192 peak_demand=65935 required_capacity=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=12 total_expected_recv=785302 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185856 peak_demand=65784 required_capacity=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=12 total_expected_recv=785880 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184704 peak_demand=65867 required_capacity=1.005051 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=12 total_expected_recv=786823 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183168 peak_demand=65857 required_capacity=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=12 total_expected_recv=787096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183296 peak_demand=65825 required_capacity=1.004410 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=12 total_expected_recv=786731 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=66155 required_capacity=1.009445 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=12 total_expected_recv=785292 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=65808 required_capacity=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=12 total_expected_recv=787295 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183424 peak_demand=65928 required_capacity=1.005981 + [2026-08-26 15:45:53] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 317.5 | throughput per GPU (TFLOP/s/GPU): 569.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.522193E+00 | loss scale: 1.0 | grad norm: 5.547 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65436 expected_overflow=0 exact_recv_per_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65737 expected_overflow=0 exact_recv_per_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65685 expected_overflow=0 exact_recv_per_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65625 expected_overflow=0 exact_recv_per_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65309 expected_overflow=0 exact_recv_per_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65499 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65436 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65737 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65529 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65685 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65309 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65625 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=81920 expected_recv=65468 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65181 expected_overflow=0 exact_recv_per_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65672 expected_overflow=0 exact_recv_per_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65181 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65684 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65413 expected_overflow=0 exact_recv_per_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65428 expected_overflow=0 exact_recv_per_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65565 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65672 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65552 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65413 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65428 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65793 expected_overflow=0 exact_recv_per_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=81920 expected_recv=65793 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65091 expected_overflow=0 exact_recv_per_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65334 expected_overflow=0 exact_recv_per_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65700 expected_overflow=0 exact_recv_per_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65887 expected_overflow=0 exact_recv_per_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65680 expected_overflow=0 exact_recv_per_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65091 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65334 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65700 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65887 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65680 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=81920 expected_recv=65526 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65698 expected_overflow=0 exact_recv_per_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65289 expected_overflow=0 exact_recv_per_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65713 expected_overflow=0 exact_recv_per_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65698 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65131 expected_overflow=0 exact_recv_per_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65289 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65717 expected_overflow=0 exact_recv_per_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65480 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65713 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65131 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65702 expected_overflow=0 exact_recv_per_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65717 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=81920 expected_recv=65702 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65486 expected_overflow=0 exact_recv_per_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65272 expected_overflow=0 exact_recv_per_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65584 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65272 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65486 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65834 expected_overflow=0 exact_recv_per_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65503 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65551 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65369 expected_overflow=0 exact_recv_per_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=81920 expected_recv=65834 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65369 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65560 expected_overflow=0 exact_recv_per_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65502 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65807 expected_overflow=0 exact_recv_per_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65475 expected_overflow=0 exact_recv_per_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65560 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65530 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65199 expected_overflow=0 exact_recv_per_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65492 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65807 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=81920 expected_recv=65475 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65199 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65637 expected_overflow=0 exact_recv_per_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65637 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65272 expected_overflow=0 exact_recv_per_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65947 expected_overflow=0 exact_recv_per_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65816 expected_overflow=0 exact_recv_per_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65272 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65626 expected_overflow=0 exact_recv_per_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65440 expected_overflow=0 exact_recv_per_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65947 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65816 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65351 expected_overflow=0 exact_recv_per_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65626 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65898 expected_overflow=0 exact_recv_per_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65440 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=81920 expected_recv=65351 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65898 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65653 expected_overflow=0 exact_recv_per_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65653 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65446 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65648 expected_overflow=0 exact_recv_per_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65092 expected_overflow=0 exact_recv_per_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65305 expected_overflow=0 exact_recv_per_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65556 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65648 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65092 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65690 expected_overflow=0 exact_recv_per_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65369 expected_overflow=0 exact_recv_per_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65305 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=81920 expected_recv=65690 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65321 expected_overflow=0 exact_recv_per_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65369 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65321 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65111 expected_overflow=0 exact_recv_per_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65111 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65829 expected_overflow=0 exact_recv_per_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65663 expected_overflow=0 exact_recv_per_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66025 expected_overflow=0 exact_recv_per_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65459 expected_overflow=0 exact_recv_per_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65829 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65663 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=81920 expected_recv=66025 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65749 expected_overflow=0 exact_recv_per_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65459 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=81920 expected_recv=65511 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65749 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65658 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65500 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65403 expected_overflow=0 exact_recv_per_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65243 expected_overflow=0 exact_recv_per_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65666 expected_overflow=0 exact_recv_per_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65403 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65795 expected_overflow=0 exact_recv_per_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65274 expected_overflow=0 exact_recv_per_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65243 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65666 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65795 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=81920 expected_recv=65274 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65427 expected_overflow=0 exact_recv_per_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65684 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65524 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65427 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65515 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65882 expected_overflow=0 exact_recv_per_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65474 expected_overflow=0 exact_recv_per_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65210 expected_overflow=0 exact_recv_per_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65882 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65474 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65210 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=81920 expected_recv=65572 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65655 expected_overflow=0 exact_recv_per_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65428 expected_overflow=0 exact_recv_per_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65686 expected_overflow=0 exact_recv_per_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65655 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65428 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65686 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65336 expected_overflow=0 exact_recv_per_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65648 expected_overflow=0 exact_recv_per_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65493 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65506 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65336 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=81920 expected_recv=65648 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[65468, 65436, 65499, 65737, 65685, 65309, 65529, 65625] avg=65536.0000 max=65737 max_rank=3 min=65309 min_rank=5 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[65793, 65684, 65181, 65565, 65552, 65428, 65672, 65413] avg=65536.0000 max=65793 max_rank=0 min=65181 min_rank=2 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[65526, 65540, 65530, 65091, 65887, 65680, 65334, 65700] avg=65536.0000 max=65887 max_rank=4 min=65091 min_rank=3 imbalance_ratio=1.005356 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65702, 65558, 65698, 65289, 65131, 65717, 65480, 65713] avg=65536.0000 max=65717 max_rank=5 min=65131 min_rank=4 imbalance_ratio=1.002762 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[65834, 65509, 65549, 65584, 65551, 65503, 65486, 65272] avg=65536.0000 max=65834 max_rank=0 min=65272 min_rank=7 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65475, 65369, 65553, 65502, 65807, 65492, 65530, 65560] avg=65536.0000 max=65807 max_rank=4 min=65369 min_rank=1 imbalance_ratio=1.004135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65351, 65199, 65637, 65272, 65440, 65626, 65816, 65947] avg=65536.0000 max=65947 max_rank=7 min=65199 min_rank=1 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[65690, 65898, 65653, 65446, 65305, 65556, 65092, 65648] avg=65536.0000 max=65898 max_rank=1 min=65092 min_rank=6 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65511, 65369, 65321, 65111, 65459, 65829, 66025, 65663] avg=65536.0000 max=66025 max_rank=6 min=65111 min_rank=3 imbalance_ratio=1.007462 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65274, 65749, 65658, 65500, 65795, 65403, 65243, 65666] avg=65536.0000 max=65795 max_rank=4 min=65243 min_rank=6 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[65572, 65684, 65524, 65427, 65210, 65515, 65882, 65474] avg=65536.0000 max=65882 max_rank=6 min=65210 min_rank=4 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[65648, 65655, 65428, 65686, 65336, 65536, 65506, 65493] avg=65536.0000 max=65686 max_rank=3 min=65336 min_rank=4 imbalance_ratio=1.002289 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=12 total_expected_recv=785210 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185856 peak_demand=65737 required_capacity=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=12 total_expected_recv=786594 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=65829 required_capacity=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=12 total_expected_recv=786650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184576 peak_demand=65898 required_capacity=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=12 total_expected_recv=786231 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184064 peak_demand=65698 required_capacity=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=12 total_expected_recv=786595 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=66025 required_capacity=1.007462 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=12 total_expected_recv=786158 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=65887 required_capacity=1.005356 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=12 total_expected_recv=786844 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=65834 required_capacity=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=12 total_expected_recv=787174 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183296 peak_demand=65947 required_capacity=1.006271 + [2026-08-26 15:45:54] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 319.1 | throughput per GPU (TFLOP/s/GPU): 566.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.429519E+00 | loss scale: 1.0 | grad norm: 5.457 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65472 expected_overflow=0 exact_recv_per_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65224 expected_overflow=0 exact_recv_per_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65802 expected_overflow=0 exact_recv_per_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65540 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65534 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65472 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65224 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65802 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65558 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65589 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=81920 expected_recv=65569 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65608 expected_overflow=0 exact_recv_per_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65806 expected_overflow=0 exact_recv_per_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65395 expected_overflow=0 exact_recv_per_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65499 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65608 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65806 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65198 expected_overflow=0 exact_recv_per_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65477 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65519 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65395 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65786 expected_overflow=0 exact_recv_per_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65198 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=81920 expected_recv=65786 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65476 expected_overflow=0 exact_recv_per_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65264 expected_overflow=0 exact_recv_per_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65392 expected_overflow=0 exact_recv_per_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65488 expected_overflow=0 exact_recv_per_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65692 expected_overflow=0 exact_recv_per_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65476 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65594 expected_overflow=0 exact_recv_per_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65264 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65392 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65488 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65692 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65594 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65813 expected_overflow=0 exact_recv_per_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65569 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=81920 expected_recv=65813 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65700 expected_overflow=0 exact_recv_per_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65447 expected_overflow=0 exact_recv_per_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65700 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65691 expected_overflow=0 exact_recv_per_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65458 expected_overflow=0 exact_recv_per_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65536 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65447 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65691 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65458 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65556 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65351 expected_overflow=0 exact_recv_per_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65549 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=81920 expected_recv=65351 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65527 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65651 expected_overflow=0 exact_recv_per_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65694 expected_overflow=0 exact_recv_per_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65651 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65590 expected_overflow=0 exact_recv_per_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65023 expected_overflow=0 exact_recv_per_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65898 expected_overflow=0 exact_recv_per_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65694 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65420 expected_overflow=0 exact_recv_per_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65590 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65485 expected_overflow=0 exact_recv_per_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65023 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65898 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65420 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=81920 expected_recv=65485 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65676 expected_overflow=0 exact_recv_per_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65676 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65581 expected_overflow=0 exact_recv_per_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65799 expected_overflow=0 exact_recv_per_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65375 expected_overflow=0 exact_recv_per_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65581 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65799 expected_overflow=0 exact_recv_per_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65799 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65288 expected_overflow=0 exact_recv_per_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65443 expected_overflow=0 exact_recv_per_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65375 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65327 expected_overflow=0 exact_recv_per_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65799 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65288 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65443 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=81920 expected_recv=65327 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65230 expected_overflow=0 exact_recv_per_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65230 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65920 expected_overflow=0 exact_recv_per_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65533 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65651 expected_overflow=0 exact_recv_per_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65920 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65206 expected_overflow=0 exact_recv_per_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65651 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65206 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65635 expected_overflow=0 exact_recv_per_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65543 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65570 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=81920 expected_recv=65635 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66112 expected_overflow=0 exact_recv_per_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=81920 expected_recv=66112 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=81920 expected_recv=64958 expected_overflow=0 exact_recv_per_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=81920 expected_recv=64958 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65312 expected_overflow=0 exact_recv_per_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65310 expected_overflow=0 exact_recv_per_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65563 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65312 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65310 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65989 expected_overflow=0 exact_recv_per_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65509 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65578 expected_overflow=0 exact_recv_per_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=81920 expected_recv=65989 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65578 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65659 expected_overflow=0 exact_recv_per_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65659 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65259 expected_overflow=0 exact_recv_per_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65799 expected_overflow=0 exact_recv_per_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65259 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65410 expected_overflow=0 exact_recv_per_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65602 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65799 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65411 expected_overflow=0 exact_recv_per_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65327 expected_overflow=0 exact_recv_per_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65410 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65411 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=81920 expected_recv=65570 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65327 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65455 expected_overflow=0 exact_recv_per_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65455 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65256 expected_overflow=0 exact_recv_per_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65986 expected_overflow=0 exact_recv_per_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65580 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65256 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65662 expected_overflow=0 exact_recv_per_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65986 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65662 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65675 expected_overflow=0 exact_recv_per_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65497 expected_overflow=0 exact_recv_per_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65525 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65675 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=81920 expected_recv=65497 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65393 expected_overflow=0 exact_recv_per_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65393 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65812 expected_overflow=0 exact_recv_per_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65795 expected_overflow=0 exact_recv_per_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65125 expected_overflow=0 exact_recv_per_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65812 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65795 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65125 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65462 expected_overflow=0 exact_recv_per_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65559 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65603 expected_overflow=0 exact_recv_per_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65467 expected_overflow=0 exact_recv_per_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65462 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65603 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=81920 expected_recv=65467 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65591 expected_overflow=0 exact_recv_per_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65591 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65663 expected_overflow=0 exact_recv_per_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65663 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65552 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65632 expected_overflow=0 exact_recv_per_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65372 expected_overflow=0 exact_recv_per_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65391 expected_overflow=0 exact_recv_per_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65632 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65484 expected_overflow=0 exact_recv_per_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65372 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65391 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=81920 expected_recv=65484 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[65569, 65802, 65540, 65224, 65558, 65534, 65472, 65589] avg=65536.0000 max=65802 max_rank=1 min=65224 min_rank=3 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[65786, 65519, 65499, 65395, 65477, 65608, 65806, 65198] avg=65536.0000 max=65806 max_rank=6 min=65198 min_rank=7 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[65813, 65692, 65476, 65594, 65392, 65264, 65488, 65569] avg=65536.0000 max=65813 max_rank=0 min=65264 min_rank=5 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[65351, 65458, 65700, 65556, 65536, 65447, 65691, 65549] avg=65536.0000 max=65700 max_rank=2 min=65351 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65485, 65023, 65527, 65898, 65651, 65694, 65590, 65420] avg=65536.0000 max=65898 max_rank=3 min=65023 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65327, 65799, 65676, 65288, 65581, 65799, 65375, 65443] avg=65536.0000 max=65799 max_rank=1 min=65288 min_rank=3 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[65635, 65651, 65230, 65543, 65533, 65920, 65206, 65570] avg=65536.0000 max=65920 max_rank=5 min=65206 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65989, 65310, 66112, 65509, 64958, 65563, 65312, 65535] avg=65536.0000 max=66112 max_rank=2 min=64958 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65570, 65799, 65578, 65410, 65659, 65259, 65602, 65411] avg=65536.0000 max=65799 max_rank=1 min=65259 min_rank=5 imbalance_ratio=1.004013 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65497, 65986, 65327, 65662, 65455, 65580, 65256, 65525] avg=65536.0000 max=65986 max_rank=1 min=65256 min_rank=6 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[65467, 65125, 65675, 65559, 65393, 65812, 65795, 65462] avg=65536.0000 max=65812 max_rank=5 min=65125 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[65484, 65632, 65603, 65372, 65591, 65663, 65552, 65391] avg=65536.0000 max=65663 max_rank=5 min=65372 min_rank=3 imbalance_ratio=1.001938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=12 total_expected_recv=787143 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183936 peak_demand=65920 required_capacity=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=12 total_expected_recv=786943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184320 peak_demand=66112 required_capacity=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=12 total_expected_recv=785784 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184832 peak_demand=65659 required_capacity=1.001877 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=12 total_expected_recv=786145 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=65806 required_capacity=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=12 total_expected_recv=786010 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=65898 required_capacity=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=12 total_expected_recv=785662 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=185472 peak_demand=65589 required_capacity=1.000809 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=12 total_expected_recv=786796 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184192 peak_demand=65986 required_capacity=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=12 total_expected_recv=786973 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184448 peak_demand=65989 required_capacity=1.006912 + [2026-08-26 15:45:54] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 317.4 | throughput per GPU (TFLOP/s/GPU): 569.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.297481E+00 | loss scale: 1.0 | grad norm: 5.375 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65818 expected_overflow=0 exact_recv_per_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65815 expected_overflow=0 exact_recv_per_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65241 expected_overflow=0 exact_recv_per_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65454 expected_overflow=0 exact_recv_per_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65581 expected_overflow=0 exact_recv_per_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65045 expected_overflow=0 exact_recv_per_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65818 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65759 expected_overflow=0 exact_recv_per_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65815 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65241 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65454 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65581 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65575 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65045 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=81920 expected_recv=65759 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65894 expected_overflow=0 exact_recv_per_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65423 expected_overflow=0 exact_recv_per_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65370 expected_overflow=0 exact_recv_per_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65894 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65423 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65370 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65482 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65633 expected_overflow=0 exact_recv_per_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65501 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65311 expected_overflow=0 exact_recv_per_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65633 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65311 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=81920 expected_recv=65674 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65295 expected_overflow=0 exact_recv_per_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65434 expected_overflow=0 exact_recv_per_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65668 expected_overflow=0 exact_recv_per_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65295 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65468 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65434 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65668 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65902 expected_overflow=0 exact_recv_per_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65502 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65902 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65462 expected_overflow=0 exact_recv_per_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65557 expected_overflow=0 exact_recv_per_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65557 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=81920 expected_recv=65462 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65389 expected_overflow=0 exact_recv_per_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65535 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65480 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65389 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65461 expected_overflow=0 exact_recv_per_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65554 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65338 expected_overflow=0 exact_recv_per_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65461 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65886 expected_overflow=0 exact_recv_per_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65338 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65645 expected_overflow=0 exact_recv_per_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65886 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=81920 expected_recv=65645 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65814 expected_overflow=0 exact_recv_per_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65325 expected_overflow=0 exact_recv_per_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65692 expected_overflow=0 exact_recv_per_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65814 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65396 expected_overflow=0 exact_recv_per_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65325 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65692 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65577 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65253 expected_overflow=0 exact_recv_per_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65802 expected_overflow=0 exact_recv_per_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65396 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65429 expected_overflow=0 exact_recv_per_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65253 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65802 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=81920 expected_recv=65429 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65638 expected_overflow=0 exact_recv_per_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65923 expected_overflow=0 exact_recv_per_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65470 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65517 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65638 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65923 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65164 expected_overflow=0 exact_recv_per_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65165 expected_overflow=0 exact_recv_per_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65439 expected_overflow=0 exact_recv_per_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65164 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65972 expected_overflow=0 exact_recv_per_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65165 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65439 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=81920 expected_recv=65972 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65324 expected_overflow=0 exact_recv_per_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65109 expected_overflow=0 exact_recv_per_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65753 expected_overflow=0 exact_recv_per_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65480 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65324 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65109 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65753 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65689 expected_overflow=0 exact_recv_per_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65858 expected_overflow=0 exact_recv_per_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65577 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65689 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65498 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=81920 expected_recv=65858 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65738 expected_overflow=0 exact_recv_per_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65506 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65738 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65630 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65553 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65315 expected_overflow=0 exact_recv_per_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65364 expected_overflow=0 exact_recv_per_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65614 expected_overflow=0 exact_recv_per_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65315 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65364 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65614 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=81920 expected_recv=65568 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65351 expected_overflow=0 exact_recv_per_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65728 expected_overflow=0 exact_recv_per_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65739 expected_overflow=0 exact_recv_per_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65703 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65351 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65728 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65739 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65215 expected_overflow=0 exact_recv_per_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65417 expected_overflow=0 exact_recv_per_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65758 expected_overflow=0 exact_recv_per_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65377 expected_overflow=0 exact_recv_per_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65215 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65417 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65758 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=81920 expected_recv=65377 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65899 expected_overflow=0 exact_recv_per_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65148 expected_overflow=0 exact_recv_per_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66015 expected_overflow=0 exact_recv_per_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65607 expected_overflow=0 exact_recv_per_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65899 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65148 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=81920 expected_recv=66015 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65607 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=67328 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65325 expected_overflow=0 exact_recv_per_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65062 expected_overflow=0 exact_recv_per_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65476 expected_overflow=0 exact_recv_per_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65756 expected_overflow=0 exact_recv_per_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65325 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65062 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65476 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=81920 expected_recv=65756 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65696 expected_overflow=0 exact_recv_per_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65461 expected_overflow=0 exact_recv_per_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65556 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65485 expected_overflow=0 exact_recv_per_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65696 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65461 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65485 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65443 expected_overflow=0 exact_recv_per_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65810 expected_overflow=0 exact_recv_per_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65443 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65482 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65810 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=81920 expected_recv=65355 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65408 expected_overflow=0 exact_recv_per_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65408 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65561 expected_overflow=0 exact_recv_per_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65748 expected_overflow=0 exact_recv_per_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65434 expected_overflow=0 exact_recv_per_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65561 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65748 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65434 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65646 expected_overflow=0 exact_recv_per_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65700 expected_overflow=0 exact_recv_per_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65319 expected_overflow=0 exact_recv_per_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65472 expected_overflow=0 exact_recv_per_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65646 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65700 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65319 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=81920 expected_recv=65472 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[65759, 65818, 65454, 65241, 65815, 65581, 65045, 65575] avg=65536.0000 max=65818 max_rank=1 min=65045 min_rank=6 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[65674, 65894, 65370, 65501, 65423, 65482, 65311, 65633] avg=65536.0000 max=65894 max_rank=1 min=65311 min_rank=6 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[65557, 65295, 65668, 65502, 65468, 65434, 65462, 65902] avg=65536.0000 max=65902 max_rank=7 min=65295 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65886, 65535, 65554, 65461, 65389, 65480, 65645, 65338] avg=65536.0000 max=65886 max_rank=0 min=65338 min_rank=7 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[65802, 65692, 65577, 65396, 65814, 65325, 65429, 65253] avg=65536.0000 max=65814 max_rank=4 min=65253 min_rank=7 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65439, 65638, 65923, 65164, 65470, 65517, 65972, 65165] avg=65536.0000 max=65972 max_rank=6 min=65164 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[65498, 65109, 65753, 65577, 65324, 65480, 65858, 65689] avg=65536.0000 max=65858 max_rank=6 min=65109 min_rank=1 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[65614, 65553, 65630, 65315, 65738, 65506, 65568, 65364] avg=65536.0000 max=65738 max_rank=4 min=65315 min_rank=3 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65758, 65739, 65351, 65215, 65728, 65703, 65377, 65417] avg=65536.0000 max=65758 max_rank=0 min=65215 min_rank=3 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65476, 65607, 65148, 65062, 66015, 65899, 65756, 65325] avg=65536.0000 max=66015 max_rank=4 min=65062 min_rank=3 imbalance_ratio=1.007309 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[65810, 65485, 65696, 65482, 65461, 65556, 65355, 65443] avg=65536.0000 max=65810 max_rank=0 min=65355 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[65319, 65434, 65561, 65700, 65748, 65408, 65472, 65646] avg=65536.0000 max=65748 max_rank=4 min=65319 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=12 total_expected_recv=786371 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184704 peak_demand=65899 required_capacity=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=12 total_expected_recv=786685 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183680 peak_demand=65923 required_capacity=1.005905 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=12 total_expected_recv=786799 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183040 peak_demand=65894 required_capacity=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=12 total_expected_recv=787393 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183552 peak_demand=66015 required_capacity=1.007309 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=12 total_expected_recv=785750 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184960 peak_demand=65902 required_capacity=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=12 total_expected_recv=784616 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=186624 peak_demand=65700 required_capacity=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=12 total_expected_recv=786250 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=184704 peak_demand=65972 required_capacity=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=12 total_expected_recv=787592 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=183296 peak_demand=65886 required_capacity=1.005341 + [2026-08-26 15:45:54] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 317.7 | throughput per GPU (TFLOP/s/GPU): 569.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.167754E+00 | loss scale: 1.0 | grad norm: 5.245 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +[after training is done] datetime: 2026-08-26 15:45:54 +W0826 15:45:54.869000 131329 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:45:54.870000 131329 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 131422 closing signal SIGTERM +W0826 15:45:54.871000 131329 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 131423 closing signal SIGTERM +W0826 15:45:54.871000 131329 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 131424 closing signal SIGTERM +W0826 15:45:54.872000 131329 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 131425 closing signal SIGTERM +W0826 15:45:54.873000 131329 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 131426 closing signal SIGTERM +W0826 15:45:54.875000 131329 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 131427 closing signal SIGTERM +W0826 15:45:54.876000 131329 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 131428 closing signal SIGTERM +W0826 15:45:54.877000 131329 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 131429 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 131329 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv index 6178ae913713b67e8a9f0249dd993445927004ed..dd360584cd4c3ce85df250850165a61dc27a7a7a 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv @@ -24,9 +24,17 @@ 7, 1 0, 4 1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 2, 4 3, 4 -4, 1 +4, 2 5, 1 6, 1 7, 1 @@ -39,21 +47,21 @@ 6, 4 7, 4 0, 4 -1, 629 +1, 4 2, 629 -3, 629 -4, 629 -5, 629 -6, 629 -7, 629 -0, 921 -1, 913 -2, 919 -3, 923 -4, 909 -5, 913 -6, 909 -7, 915 +3, 4 +4, 4 +5, 4 +6, 4 +7, 47 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 0, 1831 1, 1831 2, 1831 @@ -62,38 +70,38 @@ 5, 1831 6, 1831 7, 1831 -0, 2217 -1, 2221 -2, 2219 -3, 2221 -4, 2197 -5, 2215 -6, 2209 -7, 2219 -0, 11431 -1, 11511 -2, 11311 -3, 11811 -4, 11711 -5, 11711 -6, 11511 -7, 11511 -0, 47039 -1, 47017 -2, 47057 -3, 47021 -4, 47047 -5, 47059 -6, 47037 -7, 47051 -0, 47829 -1, 47831 -2, 47841 -3, 47803 -4, 47827 -5, 47841 -6, 47831 -7, 47841 +0, 2175 +1, 2163 +2, 2161 +3, 2185 +4, 2165 +5, 2185 +6, 2175 +7, 2175 +0, 10651 +1, 10631 +2, 10431 +3, 10431 +4, 10831 +5, 10431 +6, 10431 +7, 10431 +0, 46993 +1, 47007 +2, 47001 +3, 46955 +4, 46985 +5, 47037 +6, 47019 +7, 47007 +0, 48475 +1, 48477 +2, 48487 +3, 48449 +4, 48473 +5, 48487 +6, 48477 +7, 48487 0, 48645 1, 48647 2, 48657 @@ -110,123 +118,123 @@ 5, 48817 6, 48807 7, 48817 -0, 49285 -1, 49287 -2, 49293 -3, 49259 -4, 49259 -5, 49293 -6, 49243 -7, 49293 -0, 49577 -1, 49579 -2, 49581 -3, 49551 -4, 49551 -5, 49581 -6, 49535 -7, 49581 -0, 85197 -1, 85985 -2, 85195 -3, 85211 -4, 85211 -5, 85195 -6, 85195 -7, 85195 -0, 86697 -1, 86699 -2, 86695 -3, 86711 -4, 86711 -5, 86695 -6, 86695 -7, 86695 -0, 86697 -1, 86699 -2, 86695 -3, 86711 -4, 86711 -5, 86695 -6, 86695 -7, 86695 -0, 86697 -1, 86699 -2, 86695 -3, 86711 -4, 86711 -5, 86695 -6, 86695 -7, 86695 -0, 86709 -1, 86711 +0, 49509 +1, 49511 +2, 49513 +3, 49483 +4, 49483 +5, 49513 +6, 49467 +7, 49513 +0, 60359 +1, 60381 +2, 60377 +3, 60353 +4, 60371 +5, 60357 +6, 60337 +7, 60419 +0, 86153 +1, 86111 +2, 86147 +3, 86135 +4, 86117 +5, 86153 +6, 86125 +7, 86139 +0, 86689 +1, 86691 +2, 86707 +3, 86703 +4, 86703 +5, 86707 +6, 86687 +7, 86707 +0, 86689 +1, 86691 2, 86707 -3, 86723 -4, 86723 +3, 86703 +4, 86703 5, 86707 -6, 86707 +6, 86687 7, 86707 -0, 87163 -1, 87177 -2, 87177 -3, 87209 -4, 87189 -5, 87179 -6, 87181 -7, 87167 -0, 88409 -1, 88341 -2, 88371 -3, 88373 -4, 88387 -5, 88351 -6, 88361 -7, 88341 -0, 117865 -1, 117863 -2, 117903 -3, 117859 -4, 117915 -5, 117903 -6, 117863 -7, 117903 -0, 117865 -1, 117863 -2, 117903 -3, 117859 -4, 117915 -5, 117903 -6, 117863 -7, 117903 -0, 117865 -1, 117863 -2, 117903 -3, 117859 -4, 117915 -5, 117903 -6, 117863 -7, 117903 -0, 117865 -1, 117863 -2, 117903 -3, 117859 -4, 117915 -5, 117903 -6, 117863 -7, 117903 -0, 117865 -1, 117863 -2, 117903 -3, 117859 -4, 117915 -5, 117903 -6, 117863 -7, 117903 -0, 1709 -1, 1712 -2, 2032 -3, 117859 -4, 3312 -5, 1712 -6, 1712 -7, 1712 +0, 86955 +1, 86959 +2, 86961 +3, 87009 +4, 86979 +5, 86993 +6, 87027 +7, 87035 +0, 87999 +1, 87985 +2, 87997 +3, 87995 +4, 87955 +5, 88035 +6, 87957 +7, 87983 +0, 107875 +1, 108077 +2, 106683 +3, 107349 +4, 107589 +5, 108123 +6, 107893 +7, 106763 +0, 117877 +1, 117855 +2, 117885 +3, 117871 +4, 117927 +5, 117885 +6, 117855 +7, 117885 +0, 117877 +1, 117855 +2, 117885 +3, 117871 +4, 117927 +5, 117885 +6, 117855 +7, 117885 +0, 117877 +1, 117855 +2, 117885 +3, 117871 +4, 117927 +5, 117885 +6, 117855 +7, 117885 +0, 117877 +1, 117855 +2, 117885 +3, 117871 +4, 117927 +5, 117885 +6, 117855 +7, 117885 +0, 115519 +1, 115497 +2, 115527 +3, 115513 +4, 115569 +5, 115527 +6, 115497 +7, 115527 +0, 113839 +1, 113817 +2, 113847 +3, 113833 +4, 113889 +5, 113847 +6, 113817 +7, 113847 +0, 1 +1, 4 +2, 4 +3, 4 +4, 4 +5, 132 +6, 113817 +7, 644 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log index f2ef02b2ce407088f4f3b8b6bced62726aa24620..c3f9e33381ed72123d5785be9d94389419a29916 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log @@ -1,5 +1,5 @@ -[15:00:18] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[15:00:18] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +[15:46:01] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:46:01] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) ============================================================== QWEN3_30B / ultraep (slots/rank=16 policy=matched) E=128 K=8 H'=768 m=16 @@ -8,10 +8,12 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-UltraEP @ eaf829e7 ============================================================== -W0826 15:00:20.046000 49721 torch/distributed/run.py:766] -W0826 15:00:20.046000 49721 torch/distributed/run.py:766] ***************************************** -W0826 15:00:20.046000 49721 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 15:00:20.046000 49721 torch/distributed/run.py:766] ***************************************** +W0826 15:46:02.574000 134192 torch/distributed/run.py:766] +W0826 15:46:02.574000 134192 torch/distributed/run.py:766] ***************************************** +W0826 15:46:02.574000 134192 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:46:02.574000 134192 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 @@ -568,6 +570,7 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... @@ -581,25 +584,22 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 76 - is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 67 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 72 + is connected to [Gloo] Rank 71 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 4 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 @@ -607,17 +607,17 @@ WARNING: one_logger package is required to enable e2e metrics tracking. please g [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank6]:[W826 15:00:38.149345567 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 15:00:38.149346145 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 15:00:38.150688724 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 15:00:38.150909635 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 15:00:38.151544873 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 15:00:38.151884365 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 15:00:38.153722652 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W826 15:46:20.945608242 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:46:20.945616002 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:46:20.945683906 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:46:20.945704254 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:46:20.945724790 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:46:20.946261264 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:46:20.946478225 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' make: Nothing to be done for 'default'. @@ -625,10 +625,10 @@ make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' >>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 15:00:38.339525530 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.692 seconds -time to initialize megatron (seconds): 7.232 -[after megatron is initialized] datetime: 2026-08-26 15:00:43 +[rank0]:[W826 15:46:21.106615029 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.848 seconds +time to initialize megatron (seconds): 7.263 +[after megatron is initialized] datetime: 2026-08-26 15:46:26 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -677,6 +677,7 @@ Params for bucket 2 (49033984 elements, 49033984 padded size): module.decoder.layers.47.mlp.router.weight module.decoder.layers.47.self_attention.linear_qkv.weight module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.final_layernorm.weight module.decoder.layers.47.pre_mlp_layernorm.weight module.decoder.layers.45.self_attention.q_layernorm.weight module.decoder.layers.47.self_attention.linear_proj.weight @@ -689,7 +690,6 @@ Params for bucket 2 (49033984 elements, 49033984 padded size): module.decoder.layers.46.mlp.router.weight module.decoder.layers.46.self_attention.linear_qkv.weight module.decoder.layers.45.self_attention.k_layernorm.weight - module.decoder.final_layernorm.weight module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.46.pre_mlp_layernorm.weight Params for bucket 3 (46672384 elements, 46672384 padded size): @@ -710,28 +710,35 @@ Params for bucket 3 (46672384 elements, 46672384 padded size): module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.43.self_attention.linear_proj.weight Params for bucket 4 (49031936 elements, 49031936 padded size): - module.decoder.layers.42.pre_mlp_layernorm.weight - module.decoder.layers.42.self_attention.linear_proj.weight - module.decoder.layers.40.self_attention.k_layernorm.weight - module.decoder.layers.42.self_attention.k_layernorm.weight - module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.41.self_attention.q_layernorm.weight - module.decoder.layers.41.mlp.router.weight - module.decoder.layers.41.self_attention.linear_qkv.weight module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.41.pre_mlp_layernorm.weight - module.decoder.layers.40.self_attention.q_layernorm.weight module.decoder.layers.42.self_attention.q_layernorm.weight module.decoder.layers.41.self_attention.linear_proj.weight - module.decoder.layers.40.mlp.router.weight - module.decoder.layers.40.self_attention.linear_qkv.weight module.decoder.layers.42.mlp.router.weight module.decoder.layers.42.self_attention.linear_qkv.weight module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.k_layernorm.weight Params for bucket 5 (46672384 elements, 46672384 padded size): module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight module.decoder.layers.39.mlp.router.weight module.decoder.layers.39.self_attention.linear_qkv.weight module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight @@ -739,14 +746,14 @@ Params for bucket 5 (46672384 elements, 46672384 padded size): module.decoder.layers.38.pre_mlp_layernorm.weight module.decoder.layers.38.self_attention.k_layernorm.weight module.decoder.layers.38.self_attention.q_layernorm.weight - module.decoder.layers.39.pre_mlp_layernorm.weight - module.decoder.layers.39.self_attention.k_layernorm.weight - module.decoder.layers.39.self_attention.q_layernorm.weight - module.decoder.layers.38.mlp.router.weight - module.decoder.layers.38.self_attention.linear_qkv.weight - module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.38.self_attention.linear_proj.weight Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.k_layernorm.weight module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.36.pre_mlp_layernorm.weight module.decoder.layers.37.self_attention.q_layernorm.weight @@ -759,31 +766,31 @@ Params for bucket 6 (49031936 elements, 49031936 padded size): module.decoder.layers.37.self_attention.linear_proj.weight module.decoder.layers.35.mlp.router.weight module.decoder.layers.35.self_attention.linear_qkv.weight - module.decoder.layers.37.self_attention.k_layernorm.weight - module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.35.pre_mlp_layernorm.weight - module.decoder.layers.36.self_attention.q_layernorm.weight - module.decoder.layers.36.mlp.router.weight - module.decoder.layers.36.self_attention.linear_qkv.weight - module.decoder.layers.35.self_attention.k_layernorm.weight Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.k_layernorm.weight module.decoder.layers.35.self_attention.linear_proj.weight module.decoder.layers.34.pre_mlp_layernorm.weight - module.decoder.layers.34.self_attention.k_layernorm.weight module.decoder.layers.34.self_attention.q_layernorm.weight module.decoder.layers.33.mlp.router.weight module.decoder.layers.33.self_attention.linear_qkv.weight module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.33.self_attention.linear_proj.weight - module.decoder.layers.34.mlp.router.weight - module.decoder.layers.34.self_attention.linear_qkv.weight - module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.34.self_attention.linear_proj.weight - module.decoder.layers.33.pre_mlp_layernorm.weight - module.decoder.layers.33.self_attention.k_layernorm.weight - module.decoder.layers.33.self_attention.q_layernorm.weight Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight module.decoder.layers.31.mlp.router.weight module.decoder.layers.31.self_attention.linear_qkv.weight module.decoder.layers.30.self_attention.k_layernorm.weight @@ -796,30 +803,23 @@ Params for bucket 8 (49031936 elements, 49031936 padded size): module.decoder.layers.31.self_attention.k_layernorm.weight module.decoder.layers.32.pre_mlp_layernorm.weight module.decoder.layers.30.self_attention.q_layernorm.weight - module.decoder.layers.32.self_attention.linear_proj.weight - module.decoder.layers.30.mlp.router.weight - module.decoder.layers.30.self_attention.linear_qkv.weight - module.decoder.layers.32.self_attention.k_layernorm.weight - module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.30.pre_mlp_layernorm.weight - module.decoder.layers.31.self_attention.q_layernorm.weight Params for bucket 9 (46672384 elements, 46672384 padded size): - module.decoder.layers.29.mlp.router.weight - module.decoder.layers.29.self_attention.linear_qkv.weight - module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.29.self_attention.linear_proj.weight - module.decoder.layers.28.pre_mlp_layernorm.weight - module.decoder.layers.28.self_attention.k_layernorm.weight - module.decoder.layers.28.self_attention.q_layernorm.weight module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.29.pre_mlp_layernorm.weight module.decoder.layers.29.self_attention.k_layernorm.weight module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.pre_mlp_layernorm.weight module.decoder.layers.29.self_attention.q_layernorm.weight module.decoder.layers.28.mlp.router.weight module.decoder.layers.28.self_attention.linear_qkv.weight module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight Params for bucket 10 (49031936 elements, 49031936 padded size): module.decoder.layers.26.self_attention.q_layernorm.weight module.decoder.layers.26.mlp.router.weight @@ -846,17 +846,17 @@ Params for bucket 11 (46672384 elements, 46672384 padded size): module.decoder.layers.24.self_attention.k_layernorm.weight module.decoder.layers.24.pre_mlp_layernorm.weight module.decoder.layers.24.self_attention.q_layernorm.weight - module.decoder.layers.23.mlp.router.weight - module.decoder.layers.23.self_attention.linear_qkv.weight - module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight module.decoder.layers.24.mlp.router.weight module.decoder.layers.24.self_attention.linear_qkv.weight module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.24.self_attention.linear_proj.weight - module.decoder.layers.23.pre_mlp_layernorm.weight - module.decoder.layers.23.self_attention.k_layernorm.weight - module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight Params for bucket 12 (49031936 elements, 49031936 padded size): module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.21.pre_mlp_layernorm.weight @@ -878,22 +878,22 @@ Params for bucket 12 (49031936 elements, 49031936 padded size): module.decoder.layers.21.mlp.router.weight module.decoder.layers.21.self_attention.linear_qkv.weight Params for bucket 13 (46672384 elements, 46672384 padded size): - module.decoder.layers.19.pre_mlp_layernorm.weight module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.pre_mlp_layernorm.weight module.decoder.layers.19.self_attention.q_layernorm.weight - module.decoder.layers.18.pre_mlp_layernorm.weight - module.decoder.layers.18.self_attention.k_layernorm.weight - module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.19.mlp.router.weight module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.self_attention.linear_qkv.weight module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.19.self_attention.linear_proj.weight - module.decoder.layers.18.mlp.router.weight - module.decoder.layers.18.self_attention.linear_qkv.weight - module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight Params for bucket 14 (49031936 elements, 49031936 padded size): module.decoder.layers.16.self_attention.linear_proj.weight module.decoder.layers.15.self_attention.linear_qkv.weight @@ -916,14 +916,8 @@ Params for bucket 14 (49031936 elements, 49031936 padded size): module.decoder.layers.15.self_attention.q_layernorm.weight Params for bucket 15 (46672384 elements, 46672384 padded size): module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.15.self_attention.linear_proj.weight - module.decoder.layers.14.self_attention.k_layernorm.weight - module.decoder.layers.14.self_attention.q_layernorm.weight - module.decoder.layers.13.pre_mlp_layernorm.weight - module.decoder.layers.13.self_attention.q_layernorm.weight - module.decoder.layers.13.self_attention.linear_qkv.weight module.decoder.layers.14.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.14.self_attention.linear_qkv.weight module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.14.self_attention.linear_proj.weight @@ -931,36 +925,36 @@ Params for bucket 15 (46672384 elements, 46672384 padded size): module.decoder.layers.13.self_attention.k_layernorm.weight module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight Params for bucket 16 (49031936 elements, 49031936 padded size): - module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.k_layernorm.weight - module.decoder.layers.12.self_attention.q_layernorm.weight - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.self_attention.q_layernorm.weight - module.decoder.layers.10.self_attention.q_layernorm.weight - module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.10.self_attention.k_layernorm.weight module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.self_attention.k_layernorm.weight module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.q_layernorm.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.q_layernorm.weight Params for bucket 17 (46672384 elements, 46672384 padded size): - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.k_layernorm.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.9.self_attention.q_layernorm.weight - module.decoder.layers.8.pre_mlp_layernorm.weight - module.decoder.layers.8.self_attention.k_layernorm.weight - module.decoder.layers.8.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_proj.weight @@ -968,7 +962,22 @@ Params for bucket 17 (46672384 elements, 46672384 padded size): module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.self_attention.linear_qkv.weight @@ -979,56 +988,47 @@ Params for bucket 18 (49031936 elements, 49031936 padded size): module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.6.self_attention.k_layernorm.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.q_layernorm.weight - module.decoder.layers.7.self_attention.k_layernorm.weight - module.decoder.layers.5.self_attention.k_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.6.self_attention.q_layernorm.weight Params for bucket 19 (46672384 elements, 46672384 padded size): - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.k_layernorm.weight - module.decoder.layers.4.self_attention.q_layernorm.weight - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.router.weight + module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 20 (49031936 elements, 49031936 padded size): - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.router.weight + module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight Params for bucket 21 (111413248 elements, 111413248 padded size): module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.embedding.word_embeddings.weight module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 Params for bucket 1 (40894464 elements, 40894464 padded size): module.decoder.layers.47.mlp.experts.linear_fc2.weight13 @@ -1197,42 +1197,52 @@ Params for bucket 9 (40894464 elements, 40894464 padded size): module.decoder.layers.43.mlp.experts.linear_fc1.weight7 module.decoder.layers.43.mlp.experts.linear_fc1.weight5 Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 module.decoder.layers.42.mlp.experts.linear_fc2.weight15 module.decoder.layers.42.mlp.experts.linear_fc2.weight7 module.decoder.layers.42.mlp.experts.linear_fc2.weight2 - module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 module.decoder.layers.42.mlp.experts.linear_fc2.weight12 module.decoder.layers.42.mlp.experts.linear_fc2.weight4 module.decoder.layers.42.mlp.experts.linear_fc1.weight15 - module.decoder.layers.42.mlp.experts.linear_fc2.weight9 module.decoder.layers.42.mlp.experts.linear_fc2.weight0 - module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 module.decoder.layers.42.mlp.experts.linear_fc2.weight14 module.decoder.layers.42.mlp.experts.linear_fc2.weight6 - module.decoder.layers.43.mlp.experts.linear_fc1.weight0 module.decoder.layers.42.mlp.experts.linear_fc2.weight11 module.decoder.layers.42.mlp.experts.linear_fc2.weight3 module.decoder.layers.42.mlp.experts.linear_fc1.weight14 - module.decoder.layers.42.mlp.experts.linear_fc2.weight8 - module.decoder.layers.42.mlp.experts.linear_fc2.weight13 - module.decoder.layers.42.mlp.experts.linear_fc2.weight5 - module.decoder.layers.42.mlp.experts.linear_fc2.weight10 - module.decoder.layers.42.mlp.experts.linear_fc2.weight1 Params for bucket 11 (40894464 elements, 40894464 padded size): - module.decoder.layers.42.mlp.experts.linear_fc1.weight10 - module.decoder.layers.42.mlp.experts.linear_fc1.weight12 - module.decoder.layers.42.mlp.experts.linear_fc1.weight8 - module.decoder.layers.42.mlp.experts.linear_fc1.weight6 - module.decoder.layers.42.mlp.experts.linear_fc1.weight4 - module.decoder.layers.42.mlp.experts.linear_fc1.weight3 module.decoder.layers.42.mlp.experts.linear_fc1.weight13 - module.decoder.layers.42.mlp.experts.linear_fc1.weight9 module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 module.decoder.layers.42.mlp.experts.linear_fc1.weight7 module.decoder.layers.42.mlp.experts.linear_fc1.weight5 module.decoder.layers.42.mlp.experts.linear_fc1.weight2 module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 module.decoder.layers.41.mlp.experts.linear_fc2.weight14 module.decoder.layers.41.mlp.experts.linear_fc2.weight6 module.decoder.layers.41.mlp.experts.linear_fc2.weight11 @@ -1244,38 +1254,22 @@ Params for bucket 12 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc2.weight10 module.decoder.layers.41.mlp.experts.linear_fc2.weight1 module.decoder.layers.41.mlp.experts.linear_fc1.weight13 - module.decoder.layers.41.mlp.experts.linear_fc2.weight15 - module.decoder.layers.41.mlp.experts.linear_fc2.weight7 - module.decoder.layers.41.mlp.experts.linear_fc2.weight2 - module.decoder.layers.42.mlp.experts.linear_fc1.weight0 - module.decoder.layers.41.mlp.experts.linear_fc2.weight12 - module.decoder.layers.41.mlp.experts.linear_fc2.weight4 - module.decoder.layers.41.mlp.experts.linear_fc1.weight15 - module.decoder.layers.41.mlp.experts.linear_fc2.weight9 - module.decoder.layers.41.mlp.experts.linear_fc2.weight0 - module.decoder.layers.41.mlp.experts.linear_fc1.weight12 Params for bucket 13 (40894464 elements, 40894464 padded size): - module.decoder.layers.41.mlp.experts.linear_fc1.weight11 - module.decoder.layers.41.mlp.experts.linear_fc1.weight9 - module.decoder.layers.41.mlp.experts.linear_fc1.weight7 - module.decoder.layers.41.mlp.experts.linear_fc1.weight5 - module.decoder.layers.41.mlp.experts.linear_fc1.weight2 - module.decoder.layers.41.mlp.experts.linear_fc1.weight1 - module.decoder.layers.40.mlp.experts.linear_fc2.weight14 module.decoder.layers.41.mlp.experts.linear_fc1.weight10 module.decoder.layers.41.mlp.experts.linear_fc1.weight8 module.decoder.layers.41.mlp.experts.linear_fc1.weight6 module.decoder.layers.41.mlp.experts.linear_fc1.weight4 module.decoder.layers.41.mlp.experts.linear_fc1.weight3 module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 module.decoder.layers.40.mlp.experts.linear_fc2.weight15 Params for bucket 14 (40894464 elements, 40894464 padded size): - module.decoder.layers.40.mlp.experts.linear_fc2.weight12 - module.decoder.layers.40.mlp.experts.linear_fc2.weight4 - module.decoder.layers.40.mlp.experts.linear_fc1.weight15 - module.decoder.layers.40.mlp.experts.linear_fc2.weight9 - module.decoder.layers.40.mlp.experts.linear_fc2.weight0 - module.decoder.layers.40.mlp.experts.linear_fc1.weight12 module.decoder.layers.40.mlp.experts.linear_fc2.weight6 module.decoder.layers.40.mlp.experts.linear_fc2.weight11 module.decoder.layers.40.mlp.experts.linear_fc2.weight3 @@ -1290,35 +1284,33 @@ Params for bucket 14 (40894464 elements, 40894464 padded size): module.decoder.layers.40.mlp.experts.linear_fc2.weight7 module.decoder.layers.40.mlp.experts.linear_fc2.weight2 module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 Params for bucket 15 (40894464 elements, 40894464 padded size): module.decoder.layers.40.mlp.experts.linear_fc1.weight9 module.decoder.layers.40.mlp.experts.linear_fc1.weight7 module.decoder.layers.40.mlp.experts.linear_fc1.weight5 module.decoder.layers.40.mlp.experts.linear_fc1.weight2 module.decoder.layers.40.mlp.experts.linear_fc1.weight1 - module.decoder.layers.39.mlp.experts.linear_fc2.weight14 - module.decoder.layers.39.mlp.experts.linear_fc2.weight12 - module.decoder.layers.39.mlp.experts.linear_fc2.weight10 - module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 module.decoder.layers.40.mlp.experts.linear_fc1.weight4 module.decoder.layers.40.mlp.experts.linear_fc1.weight3 module.decoder.layers.40.mlp.experts.linear_fc1.weight0 - module.decoder.layers.39.mlp.experts.linear_fc2.weight15 - module.decoder.layers.39.mlp.experts.linear_fc2.weight13 - module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 Params for bucket 16 (40894464 elements, 40894464 padded size): - module.decoder.layers.39.mlp.experts.linear_fc2.weight8 - module.decoder.layers.39.mlp.experts.linear_fc2.weight6 - module.decoder.layers.39.mlp.experts.linear_fc2.weight4 - module.decoder.layers.39.mlp.experts.linear_fc2.weight1 - module.decoder.layers.39.mlp.experts.linear_fc1.weight15 - module.decoder.layers.39.mlp.experts.linear_fc1.weight13 - module.decoder.layers.39.mlp.experts.linear_fc1.weight11 - module.decoder.layers.39.mlp.experts.linear_fc1.weight9 module.decoder.layers.39.mlp.experts.linear_fc2.weight9 - module.decoder.layers.39.mlp.experts.linear_fc2.weight5 module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 module.decoder.layers.39.mlp.experts.linear_fc2.weight3 module.decoder.layers.39.mlp.experts.linear_fc2.weight2 module.decoder.layers.39.mlp.experts.linear_fc2.weight0 @@ -1326,7 +1318,24 @@ Params for bucket 16 (40894464 elements, 40894464 padded size): module.decoder.layers.39.mlp.experts.linear_fc1.weight12 module.decoder.layers.39.mlp.experts.linear_fc1.weight10 module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 module.decoder.layers.39.mlp.experts.linear_fc1.weight7 module.decoder.layers.39.mlp.experts.linear_fc1.weight5 module.decoder.layers.39.mlp.experts.linear_fc1.weight2 @@ -1336,16 +1345,14 @@ Params for bucket 17 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc2.weight11 module.decoder.layers.38.mlp.experts.linear_fc2.weight9 module.decoder.layers.38.mlp.experts.linear_fc2.weight7 - module.decoder.layers.39.mlp.experts.linear_fc1.weight6 - module.decoder.layers.39.mlp.experts.linear_fc1.weight3 - module.decoder.layers.39.mlp.experts.linear_fc1.weight4 - module.decoder.layers.39.mlp.experts.linear_fc1.weight0 - module.decoder.layers.38.mlp.experts.linear_fc2.weight14 - module.decoder.layers.38.mlp.experts.linear_fc2.weight12 - module.decoder.layers.38.mlp.experts.linear_fc2.weight10 - module.decoder.layers.38.mlp.experts.linear_fc2.weight8 - module.decoder.layers.38.mlp.experts.linear_fc2.weight6 Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 module.decoder.layers.38.mlp.experts.linear_fc2.weight5 module.decoder.layers.38.mlp.experts.linear_fc2.weight2 module.decoder.layers.38.mlp.experts.linear_fc2.weight3 @@ -1355,14 +1362,14 @@ Params for bucket 18 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc1.weight10 module.decoder.layers.38.mlp.experts.linear_fc1.weight8 module.decoder.layers.38.mlp.experts.linear_fc1.weight6 - module.decoder.layers.38.mlp.experts.linear_fc2.weight4 - module.decoder.layers.38.mlp.experts.linear_fc2.weight1 - module.decoder.layers.38.mlp.experts.linear_fc1.weight15 - module.decoder.layers.38.mlp.experts.linear_fc1.weight13 - module.decoder.layers.38.mlp.experts.linear_fc1.weight11 - module.decoder.layers.38.mlp.experts.linear_fc1.weight9 - module.decoder.layers.38.mlp.experts.linear_fc1.weight7 Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 module.decoder.layers.38.mlp.experts.linear_fc1.weight4 module.decoder.layers.37.mlp.experts.linear_fc2.weight8 module.decoder.layers.38.mlp.experts.linear_fc1.weight2 @@ -1376,21 +1383,7 @@ Params for bucket 19 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc1.weight0 module.decoder.layers.37.mlp.experts.linear_fc2.weight12 module.decoder.layers.37.mlp.experts.linear_fc2.weight4 - module.decoder.layers.38.mlp.experts.linear_fc1.weight5 - module.decoder.layers.37.mlp.experts.linear_fc2.weight9 - module.decoder.layers.38.mlp.experts.linear_fc1.weight3 - module.decoder.layers.37.mlp.experts.linear_fc2.weight14 - module.decoder.layers.37.mlp.experts.linear_fc2.weight6 - module.decoder.layers.37.mlp.experts.linear_fc2.weight11 - module.decoder.layers.37.mlp.experts.linear_fc2.weight3 Params for bucket 20 (40894464 elements, 40894464 padded size): - module.decoder.layers.37.mlp.experts.linear_fc2.weight1 - module.decoder.layers.37.mlp.experts.linear_fc1.weight13 - module.decoder.layers.37.mlp.experts.linear_fc1.weight15 - module.decoder.layers.37.mlp.experts.linear_fc1.weight11 - module.decoder.layers.37.mlp.experts.linear_fc1.weight9 - module.decoder.layers.37.mlp.experts.linear_fc1.weight7 - module.decoder.layers.37.mlp.experts.linear_fc1.weight5 module.decoder.layers.37.mlp.experts.linear_fc2.weight0 module.decoder.layers.37.mlp.experts.linear_fc1.weight14 module.decoder.layers.37.mlp.experts.linear_fc1.weight12 @@ -1398,9 +1391,22 @@ Params for bucket 20 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc1.weight8 module.decoder.layers.37.mlp.experts.linear_fc1.weight6 module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 Params for bucket 21 (40894464 elements, 40894464 padded size): - module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 module.decoder.layers.36.mlp.experts.linear_fc2.weight7 module.decoder.layers.36.mlp.experts.linear_fc2.weight2 module.decoder.layers.37.mlp.experts.linear_fc1.weight0 @@ -1414,27 +1420,31 @@ Params for bucket 21 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc2.weight6 module.decoder.layers.36.mlp.experts.linear_fc2.weight11 module.decoder.layers.36.mlp.experts.linear_fc2.weight3 - module.decoder.layers.36.mlp.experts.linear_fc2.weight8 - module.decoder.layers.37.mlp.experts.linear_fc1.weight2 - module.decoder.layers.36.mlp.experts.linear_fc2.weight13 - module.decoder.layers.36.mlp.experts.linear_fc2.weight5 - module.decoder.layers.36.mlp.experts.linear_fc2.weight10 - module.decoder.layers.36.mlp.experts.linear_fc2.weight1 Params for bucket 22 (40894464 elements, 40894464 padded size): - module.decoder.layers.36.mlp.experts.linear_fc1.weight14 - module.decoder.layers.36.mlp.experts.linear_fc1.weight12 - module.decoder.layers.36.mlp.experts.linear_fc1.weight10 - module.decoder.layers.36.mlp.experts.linear_fc1.weight8 - module.decoder.layers.36.mlp.experts.linear_fc1.weight6 - module.decoder.layers.36.mlp.experts.linear_fc1.weight4 - module.decoder.layers.36.mlp.experts.linear_fc1.weight3 module.decoder.layers.36.mlp.experts.linear_fc1.weight13 module.decoder.layers.36.mlp.experts.linear_fc1.weight11 module.decoder.layers.36.mlp.experts.linear_fc1.weight9 module.decoder.layers.36.mlp.experts.linear_fc1.weight7 module.decoder.layers.36.mlp.experts.linear_fc1.weight5 module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 module.decoder.layers.35.mlp.experts.linear_fc2.weight14 module.decoder.layers.35.mlp.experts.linear_fc2.weight6 module.decoder.layers.35.mlp.experts.linear_fc2.weight11 @@ -1446,23 +1456,7 @@ Params for bucket 23 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc2.weight10 module.decoder.layers.35.mlp.experts.linear_fc2.weight1 module.decoder.layers.35.mlp.experts.linear_fc1.weight13 - module.decoder.layers.36.mlp.experts.linear_fc1.weight1 - module.decoder.layers.35.mlp.experts.linear_fc2.weight15 - module.decoder.layers.35.mlp.experts.linear_fc2.weight7 - module.decoder.layers.35.mlp.experts.linear_fc2.weight2 - module.decoder.layers.36.mlp.experts.linear_fc1.weight0 - module.decoder.layers.35.mlp.experts.linear_fc2.weight12 - module.decoder.layers.35.mlp.experts.linear_fc2.weight4 - module.decoder.layers.35.mlp.experts.linear_fc1.weight15 - module.decoder.layers.35.mlp.experts.linear_fc2.weight9 - module.decoder.layers.35.mlp.experts.linear_fc2.weight0 Params for bucket 24 (40894464 elements, 40894464 padded size): - module.decoder.layers.35.mlp.experts.linear_fc1.weight11 - module.decoder.layers.35.mlp.experts.linear_fc1.weight9 - module.decoder.layers.35.mlp.experts.linear_fc1.weight7 - module.decoder.layers.35.mlp.experts.linear_fc1.weight5 - module.decoder.layers.35.mlp.experts.linear_fc1.weight2 - module.decoder.layers.35.mlp.experts.linear_fc1.weight1 module.decoder.layers.35.mlp.experts.linear_fc1.weight12 module.decoder.layers.35.mlp.experts.linear_fc1.weight8 module.decoder.layers.35.mlp.experts.linear_fc1.weight10 @@ -1470,7 +1464,20 @@ Params for bucket 24 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc1.weight4 module.decoder.layers.35.mlp.experts.linear_fc1.weight3 module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 module.decoder.layers.34.mlp.experts.linear_fc2.weight13 module.decoder.layers.34.mlp.experts.linear_fc2.weight5 module.decoder.layers.34.mlp.experts.linear_fc2.weight10 @@ -1485,14 +1492,14 @@ Params for bucket 25 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc2.weight9 module.decoder.layers.34.mlp.experts.linear_fc2.weight0 module.decoder.layers.34.mlp.experts.linear_fc1.weight12 - module.decoder.layers.34.mlp.experts.linear_fc2.weight14 - module.decoder.layers.34.mlp.experts.linear_fc2.weight6 - module.decoder.layers.34.mlp.experts.linear_fc2.weight11 - module.decoder.layers.34.mlp.experts.linear_fc2.weight3 - module.decoder.layers.34.mlp.experts.linear_fc1.weight14 - module.decoder.layers.34.mlp.experts.linear_fc2.weight8 - module.decoder.layers.34.mlp.experts.linear_fc1.weight11 Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 module.decoder.layers.34.mlp.experts.linear_fc1.weight10 module.decoder.layers.34.mlp.experts.linear_fc1.weight8 module.decoder.layers.34.mlp.experts.linear_fc1.weight6 @@ -1501,14 +1508,14 @@ Params for bucket 26 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc1.weight0 module.decoder.layers.33.mlp.experts.linear_fc2.weight14 module.decoder.layers.33.mlp.experts.linear_fc2.weight12 - module.decoder.layers.34.mlp.experts.linear_fc1.weight7 - module.decoder.layers.34.mlp.experts.linear_fc1.weight9 - module.decoder.layers.34.mlp.experts.linear_fc1.weight5 - module.decoder.layers.34.mlp.experts.linear_fc1.weight2 - module.decoder.layers.34.mlp.experts.linear_fc1.weight1 - module.decoder.layers.33.mlp.experts.linear_fc2.weight15 - module.decoder.layers.33.mlp.experts.linear_fc2.weight13 Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 module.decoder.layers.33.mlp.experts.linear_fc2.weight4 module.decoder.layers.33.mlp.experts.linear_fc1.weight15 module.decoder.layers.33.mlp.experts.linear_fc2.weight9 @@ -1521,41 +1528,25 @@ Params for bucket 27 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc1.weight14 module.decoder.layers.33.mlp.experts.linear_fc2.weight8 module.decoder.layers.33.mlp.experts.linear_fc1.weight11 - module.decoder.layers.33.mlp.experts.linear_fc2.weight5 - module.decoder.layers.33.mlp.experts.linear_fc2.weight10 - module.decoder.layers.33.mlp.experts.linear_fc2.weight1 - module.decoder.layers.33.mlp.experts.linear_fc1.weight13 - module.decoder.layers.33.mlp.experts.linear_fc2.weight7 - module.decoder.layers.33.mlp.experts.linear_fc2.weight2 - module.decoder.layers.33.mlp.experts.linear_fc1.weight10 Params for bucket 28 (40894464 elements, 40894464 padded size): - module.decoder.layers.33.mlp.experts.linear_fc1.weight5 - module.decoder.layers.33.mlp.experts.linear_fc1.weight7 - module.decoder.layers.33.mlp.experts.linear_fc1.weight2 - module.decoder.layers.33.mlp.experts.linear_fc1.weight1 - module.decoder.layers.32.mlp.experts.linear_fc2.weight15 - module.decoder.layers.32.mlp.experts.linear_fc2.weight13 - module.decoder.layers.32.mlp.experts.linear_fc2.weight11 - module.decoder.layers.32.mlp.experts.linear_fc2.weight9 module.decoder.layers.33.mlp.experts.linear_fc1.weight8 - module.decoder.layers.33.mlp.experts.linear_fc1.weight6 module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 module.decoder.layers.33.mlp.experts.linear_fc1.weight3 module.decoder.layers.33.mlp.experts.linear_fc1.weight0 module.decoder.layers.32.mlp.experts.linear_fc2.weight14 module.decoder.layers.32.mlp.experts.linear_fc2.weight12 module.decoder.layers.32.mlp.experts.linear_fc2.weight10 module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 Params for bucket 29 (40894464 elements, 40894464 padded size): - module.decoder.layers.32.mlp.experts.linear_fc2.weight7 - module.decoder.layers.32.mlp.experts.linear_fc2.weight5 - module.decoder.layers.32.mlp.experts.linear_fc2.weight3 - module.decoder.layers.32.mlp.experts.linear_fc2.weight2 - module.decoder.layers.32.mlp.experts.linear_fc2.weight0 - module.decoder.layers.32.mlp.experts.linear_fc1.weight14 - module.decoder.layers.32.mlp.experts.linear_fc1.weight12 - module.decoder.layers.32.mlp.experts.linear_fc1.weight10 - module.decoder.layers.32.mlp.experts.linear_fc1.weight8 module.decoder.layers.32.mlp.experts.linear_fc2.weight6 module.decoder.layers.32.mlp.experts.linear_fc2.weight4 module.decoder.layers.32.mlp.experts.linear_fc2.weight1 @@ -1564,7 +1555,22 @@ Params for bucket 29 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc1.weight11 module.decoder.layers.32.mlp.experts.linear_fc1.weight9 module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 module.decoder.layers.32.mlp.experts.linear_fc1.weight6 module.decoder.layers.31.mlp.experts.linear_fc2.weight10 module.decoder.layers.32.mlp.experts.linear_fc1.weight1 @@ -1578,29 +1584,32 @@ Params for bucket 30 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc1.weight3 module.decoder.layers.31.mlp.experts.linear_fc2.weight14 module.decoder.layers.31.mlp.experts.linear_fc2.weight6 - module.decoder.layers.31.mlp.experts.linear_fc2.weight11 - module.decoder.layers.32.mlp.experts.linear_fc1.weight4 - module.decoder.layers.31.mlp.experts.linear_fc2.weight8 - module.decoder.layers.32.mlp.experts.linear_fc1.weight2 - module.decoder.layers.31.mlp.experts.linear_fc2.weight13 - module.decoder.layers.31.mlp.experts.linear_fc2.weight5 Params for bucket 31 (40894464 elements, 40894464 padded size): - module.decoder.layers.31.mlp.experts.linear_fc2.weight1 - module.decoder.layers.31.mlp.experts.linear_fc1.weight15 - module.decoder.layers.31.mlp.experts.linear_fc1.weight13 - module.decoder.layers.31.mlp.experts.linear_fc1.weight11 - module.decoder.layers.31.mlp.experts.linear_fc1.weight9 - module.decoder.layers.31.mlp.experts.linear_fc1.weight7 - module.decoder.layers.31.mlp.experts.linear_fc1.weight5 module.decoder.layers.31.mlp.experts.linear_fc2.weight3 - module.decoder.layers.31.mlp.experts.linear_fc2.weight0 module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 module.decoder.layers.31.mlp.experts.linear_fc1.weight14 module.decoder.layers.31.mlp.experts.linear_fc1.weight12 module.decoder.layers.31.mlp.experts.linear_fc1.weight10 module.decoder.layers.31.mlp.experts.linear_fc1.weight8 module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 module.decoder.layers.30.mlp.experts.linear_fc2.weight9 module.decoder.layers.30.mlp.experts.linear_fc2.weight0 module.decoder.layers.31.mlp.experts.linear_fc1.weight3 @@ -1613,66 +1622,63 @@ Params for bucket 32 (40894464 elements, 40894464 padded size): module.decoder.layers.31.mlp.experts.linear_fc1.weight2 module.decoder.layers.30.mlp.experts.linear_fc2.weight13 module.decoder.layers.30.mlp.experts.linear_fc2.weight5 - module.decoder.layers.30.mlp.experts.linear_fc2.weight10 - module.decoder.layers.30.mlp.experts.linear_fc2.weight1 - module.decoder.layers.31.mlp.experts.linear_fc1.weight1 - module.decoder.layers.30.mlp.experts.linear_fc2.weight15 - module.decoder.layers.30.mlp.experts.linear_fc2.weight7 - module.decoder.layers.30.mlp.experts.linear_fc2.weight2 - module.decoder.layers.31.mlp.experts.linear_fc1.weight0 - module.decoder.layers.30.mlp.experts.linear_fc2.weight12 - module.decoder.layers.30.mlp.experts.linear_fc2.weight4 Params for bucket 33 (40894464 elements, 40894464 padded size): - module.decoder.layers.30.mlp.experts.linear_fc1.weight12 - module.decoder.layers.30.mlp.experts.linear_fc1.weight14 - module.decoder.layers.30.mlp.experts.linear_fc1.weight10 - module.decoder.layers.30.mlp.experts.linear_fc1.weight8 - module.decoder.layers.30.mlp.experts.linear_fc1.weight6 - module.decoder.layers.30.mlp.experts.linear_fc1.weight4 - module.decoder.layers.30.mlp.experts.linear_fc1.weight3 module.decoder.layers.30.mlp.experts.linear_fc1.weight15 module.decoder.layers.30.mlp.experts.linear_fc1.weight13 module.decoder.layers.30.mlp.experts.linear_fc1.weight11 module.decoder.layers.30.mlp.experts.linear_fc1.weight9 module.decoder.layers.30.mlp.experts.linear_fc1.weight7 module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 module.decoder.layers.29.mlp.experts.linear_fc2.weight8 module.decoder.layers.30.mlp.experts.linear_fc1.weight2 module.decoder.layers.29.mlp.experts.linear_fc2.weight13 module.decoder.layers.29.mlp.experts.linear_fc2.weight5 module.decoder.layers.29.mlp.experts.linear_fc2.weight10 module.decoder.layers.29.mlp.experts.linear_fc2.weight1 - module.decoder.layers.30.mlp.experts.linear_fc1.weight1 module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 module.decoder.layers.29.mlp.experts.linear_fc2.weight7 module.decoder.layers.29.mlp.experts.linear_fc2.weight2 module.decoder.layers.30.mlp.experts.linear_fc1.weight0 module.decoder.layers.29.mlp.experts.linear_fc2.weight12 module.decoder.layers.29.mlp.experts.linear_fc2.weight4 module.decoder.layers.29.mlp.experts.linear_fc1.weight15 - module.decoder.layers.29.mlp.experts.linear_fc2.weight9 - module.decoder.layers.29.mlp.experts.linear_fc2.weight0 - module.decoder.layers.29.mlp.experts.linear_fc2.weight14 - module.decoder.layers.29.mlp.experts.linear_fc2.weight6 - module.decoder.layers.29.mlp.experts.linear_fc2.weight11 - module.decoder.layers.29.mlp.experts.linear_fc2.weight3 - module.decoder.layers.29.mlp.experts.linear_fc1.weight14 Params for bucket 35 (40894464 elements, 40894464 padded size): - module.decoder.layers.29.mlp.experts.linear_fc1.weight13 - module.decoder.layers.29.mlp.experts.linear_fc1.weight11 - module.decoder.layers.29.mlp.experts.linear_fc1.weight9 - module.decoder.layers.29.mlp.experts.linear_fc1.weight7 - module.decoder.layers.29.mlp.experts.linear_fc1.weight5 - module.decoder.layers.29.mlp.experts.linear_fc1.weight2 - module.decoder.layers.29.mlp.experts.linear_fc1.weight1 module.decoder.layers.29.mlp.experts.linear_fc1.weight12 module.decoder.layers.29.mlp.experts.linear_fc1.weight10 module.decoder.layers.29.mlp.experts.linear_fc1.weight8 module.decoder.layers.29.mlp.experts.linear_fc1.weight6 module.decoder.layers.29.mlp.experts.linear_fc1.weight4 module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 module.decoder.layers.28.mlp.experts.linear_fc2.weight15 module.decoder.layers.28.mlp.experts.linear_fc2.weight7 module.decoder.layers.28.mlp.experts.linear_fc2.weight2 @@ -1688,47 +1694,41 @@ Params for bucket 36 (40894464 elements, 40894464 padded size): module.decoder.layers.28.mlp.experts.linear_fc2.weight11 module.decoder.layers.28.mlp.experts.linear_fc2.weight3 module.decoder.layers.28.mlp.experts.linear_fc1.weight14 - module.decoder.layers.28.mlp.experts.linear_fc2.weight8 - module.decoder.layers.28.mlp.experts.linear_fc2.weight13 - module.decoder.layers.28.mlp.experts.linear_fc2.weight5 - module.decoder.layers.28.mlp.experts.linear_fc2.weight10 - module.decoder.layers.28.mlp.experts.linear_fc2.weight1 - module.decoder.layers.28.mlp.experts.linear_fc1.weight13 Params for bucket 37 (40894464 elements, 40894464 padded size): - module.decoder.layers.28.mlp.experts.linear_fc1.weight10 - module.decoder.layers.28.mlp.experts.linear_fc1.weight8 - module.decoder.layers.28.mlp.experts.linear_fc1.weight6 - module.decoder.layers.28.mlp.experts.linear_fc1.weight4 - module.decoder.layers.28.mlp.experts.linear_fc1.weight3 - module.decoder.layers.28.mlp.experts.linear_fc1.weight0 - module.decoder.layers.27.mlp.experts.linear_fc2.weight14 module.decoder.layers.28.mlp.experts.linear_fc1.weight11 - module.decoder.layers.28.mlp.experts.linear_fc1.weight7 module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 module.decoder.layers.28.mlp.experts.linear_fc1.weight5 module.decoder.layers.28.mlp.experts.linear_fc1.weight2 module.decoder.layers.28.mlp.experts.linear_fc1.weight1 module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 module.decoder.layers.27.mlp.experts.linear_fc2.weight6 module.decoder.layers.27.mlp.experts.linear_fc2.weight11 module.decoder.layers.27.mlp.experts.linear_fc2.weight3 - module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 module.decoder.layers.27.mlp.experts.linear_fc2.weight8 - module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 module.decoder.layers.27.mlp.experts.linear_fc2.weight13 module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 module.decoder.layers.27.mlp.experts.linear_fc2.weight10 module.decoder.layers.27.mlp.experts.linear_fc2.weight1 - module.decoder.layers.27.mlp.experts.linear_fc1.weight13 - module.decoder.layers.27.mlp.experts.linear_fc2.weight7 - module.decoder.layers.27.mlp.experts.linear_fc2.weight2 - module.decoder.layers.27.mlp.experts.linear_fc1.weight10 - module.decoder.layers.27.mlp.experts.linear_fc2.weight12 - module.decoder.layers.27.mlp.experts.linear_fc2.weight4 - module.decoder.layers.27.mlp.experts.linear_fc1.weight15 - module.decoder.layers.27.mlp.experts.linear_fc2.weight9 - module.decoder.layers.27.mlp.experts.linear_fc2.weight0 module.decoder.layers.27.mlp.experts.linear_fc1.weight12 Params for bucket 39 (40894464 elements, 40894464 padded size): module.decoder.layers.27.mlp.experts.linear_fc1.weight9 @@ -1839,34 +1839,28 @@ Params for bucket 44 (40894464 elements, 40894464 padded size): module.decoder.layers.24.mlp.experts.linear_fc1.weight7 module.decoder.layers.24.mlp.experts.linear_fc1.weight5 Params for bucket 45 (40894464 elements, 40894464 padded size): - module.decoder.layers.23.mlp.experts.linear_fc2.weight10 - module.decoder.layers.23.mlp.experts.linear_fc2.weight1 - module.decoder.layers.24.mlp.experts.linear_fc1.weight1 - module.decoder.layers.23.mlp.experts.linear_fc2.weight15 - module.decoder.layers.23.mlp.experts.linear_fc2.weight7 - module.decoder.layers.23.mlp.experts.linear_fc2.weight2 - module.decoder.layers.24.mlp.experts.linear_fc1.weight0 - module.decoder.layers.23.mlp.experts.linear_fc2.weight12 - module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight15 module.decoder.layers.23.mlp.experts.linear_fc2.weight9 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 module.decoder.layers.23.mlp.experts.linear_fc2.weight14 module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 module.decoder.layers.23.mlp.experts.linear_fc2.weight11 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight8 - module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight13 module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 Params for bucket 46 (40894464 elements, 40894464 padded size): - module.decoder.layers.23.mlp.experts.linear_fc1.weight13 - module.decoder.layers.23.mlp.experts.linear_fc1.weight11 - module.decoder.layers.23.mlp.experts.linear_fc1.weight9 - module.decoder.layers.23.mlp.experts.linear_fc1.weight7 - module.decoder.layers.23.mlp.experts.linear_fc1.weight5 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc1.weight14 module.decoder.layers.23.mlp.experts.linear_fc1.weight12 module.decoder.layers.23.mlp.experts.linear_fc1.weight10 @@ -1874,6 +1868,12 @@ Params for bucket 46 (40894464 elements, 40894464 padded size): module.decoder.layers.23.mlp.experts.linear_fc1.weight6 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 Params for bucket 47 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc2.weight8 module.decoder.layers.22.mlp.experts.linear_fc2.weight13 @@ -1881,18 +1881,18 @@ Params for bucket 47 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc2.weight10 module.decoder.layers.22.mlp.experts.linear_fc2.weight1 module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight15 module.decoder.layers.22.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight12 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight15 module.decoder.layers.22.mlp.experts.linear_fc2.weight9 module.decoder.layers.22.mlp.experts.linear_fc2.weight0 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight14 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 - module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight11 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc1.weight14 @@ -2006,33 +2006,25 @@ Params for bucket 53 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc1.weight7 Params for bucket 54 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc1.weight6 - module.decoder.layers.18.mlp.experts.linear_fc2.weight9 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.18.mlp.experts.linear_fc2.weight14 - module.decoder.layers.18.mlp.experts.linear_fc2.weight6 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 - module.decoder.layers.18.mlp.experts.linear_fc2.weight11 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight8 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 module.decoder.layers.18.mlp.experts.linear_fc2.weight13 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.18.mlp.experts.linear_fc2.weight10 - module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight15 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight12 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 Params for bucket 55 (40894464 elements, 40894464 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight3 - module.decoder.layers.18.mlp.experts.linear_fc2.weight0 - module.decoder.layers.18.mlp.experts.linear_fc2.weight2 - module.decoder.layers.18.mlp.experts.linear_fc1.weight14 - module.decoder.layers.18.mlp.experts.linear_fc1.weight12 - module.decoder.layers.18.mlp.experts.linear_fc1.weight10 - module.decoder.layers.18.mlp.experts.linear_fc1.weight8 - module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc1.weight15 module.decoder.layers.18.mlp.experts.linear_fc2.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight13 @@ -2040,16 +2032,15 @@ Params for bucket 55 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight9 module.decoder.layers.18.mlp.experts.linear_fc1.weight7 module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 Params for bucket 56 (40894464 elements, 40894464 padded size): - module.decoder.layers.18.mlp.experts.linear_fc1.weight4 - module.decoder.layers.17.mlp.experts.linear_fc2.weight15 - module.decoder.layers.17.mlp.experts.linear_fc2.weight7 - module.decoder.layers.17.mlp.experts.linear_fc2.weight2 - module.decoder.layers.18.mlp.experts.linear_fc1.weight2 - module.decoder.layers.17.mlp.experts.linear_fc2.weight12 - module.decoder.layers.17.mlp.experts.linear_fc2.weight4 - module.decoder.layers.17.mlp.experts.linear_fc2.weight9 - module.decoder.layers.17.mlp.experts.linear_fc2.weight0 module.decoder.layers.18.mlp.experts.linear_fc1.weight1 module.decoder.layers.17.mlp.experts.linear_fc2.weight14 module.decoder.layers.17.mlp.experts.linear_fc2.weight6 @@ -2062,7 +2053,22 @@ Params for bucket 56 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.17.mlp.experts.linear_fc2.weight10 module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 module.decoder.layers.17.mlp.experts.linear_fc1.weight14 module.decoder.layers.17.mlp.experts.linear_fc1.weight12 module.decoder.layers.17.mlp.experts.linear_fc1.weight10 @@ -2070,29 +2076,23 @@ Params for bucket 57 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight6 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight3 - module.decoder.layers.17.mlp.experts.linear_fc1.weight15 - module.decoder.layers.17.mlp.experts.linear_fc1.weight11 - module.decoder.layers.17.mlp.experts.linear_fc1.weight13 - module.decoder.layers.17.mlp.experts.linear_fc1.weight9 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight12 module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc1.weight15 module.decoder.layers.16.mlp.experts.linear_fc2.weight9 module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 module.decoder.layers.16.mlp.experts.linear_fc2.weight14 module.decoder.layers.16.mlp.experts.linear_fc2.weight6 - module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 module.decoder.layers.16.mlp.experts.linear_fc2.weight11 module.decoder.layers.16.mlp.experts.linear_fc2.weight3 module.decoder.layers.16.mlp.experts.linear_fc1.weight14 module.decoder.layers.16.mlp.experts.linear_fc2.weight8 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 module.decoder.layers.16.mlp.experts.linear_fc2.weight13 module.decoder.layers.16.mlp.experts.linear_fc2.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight0 module.decoder.layers.16.mlp.experts.linear_fc2.weight10 module.decoder.layers.16.mlp.experts.linear_fc2.weight1 module.decoder.layers.16.mlp.experts.linear_fc2.weight15 @@ -2151,34 +2151,26 @@ Params for bucket 61 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc2.weight15 Params for bucket 62 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc2.weight8 - module.decoder.layers.14.mlp.experts.linear_fc2.weight2 - module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 module.decoder.layers.14.mlp.experts.linear_fc2.weight13 module.decoder.layers.14.mlp.experts.linear_fc2.weight5 - module.decoder.layers.14.mlp.experts.linear_fc1.weight15 module.decoder.layers.14.mlp.experts.linear_fc2.weight10 module.decoder.layers.14.mlp.experts.linear_fc2.weight1 - module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 module.decoder.layers.14.mlp.experts.linear_fc2.weight12 module.decoder.layers.14.mlp.experts.linear_fc2.weight4 - module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 module.decoder.layers.14.mlp.experts.linear_fc2.weight9 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 - module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 module.decoder.layers.14.mlp.experts.linear_fc2.weight6 module.decoder.layers.14.mlp.experts.linear_fc2.weight11 module.decoder.layers.14.mlp.experts.linear_fc2.weight3 - module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 Params for bucket 63 (40894464 elements, 40894464 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 - module.decoder.layers.14.mlp.experts.linear_fc1.weight8 - module.decoder.layers.14.mlp.experts.linear_fc1.weight4 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight15 - module.decoder.layers.13.mlp.experts.linear_fc2.weight13 - module.decoder.layers.13.mlp.experts.linear_fc2.weight11 module.decoder.layers.14.mlp.experts.linear_fc1.weight9 module.decoder.layers.14.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 @@ -2187,10 +2179,26 @@ Params for bucket 63 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight14 module.decoder.layers.13.mlp.experts.linear_fc2.weight12 module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 module.decoder.layers.13.mlp.experts.linear_fc2.weight9 - module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight3 module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight0 @@ -2198,24 +2206,7 @@ Params for bucket 64 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight12 module.decoder.layers.13.mlp.experts.linear_fc1.weight10 module.decoder.layers.13.mlp.experts.linear_fc1.weight8 - module.decoder.layers.13.mlp.experts.linear_fc2.weight8 - module.decoder.layers.13.mlp.experts.linear_fc2.weight6 - module.decoder.layers.13.mlp.experts.linear_fc2.weight4 - module.decoder.layers.13.mlp.experts.linear_fc2.weight1 - module.decoder.layers.13.mlp.experts.linear_fc1.weight15 - module.decoder.layers.13.mlp.experts.linear_fc1.weight13 - module.decoder.layers.13.mlp.experts.linear_fc1.weight11 - module.decoder.layers.13.mlp.experts.linear_fc1.weight9 Params for bucket 65 (40894464 elements, 40894464 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight4 - module.decoder.layers.13.mlp.experts.linear_fc1.weight3 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 - module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight15 - module.decoder.layers.12.mlp.experts.linear_fc2.weight13 - module.decoder.layers.12.mlp.experts.linear_fc2.weight11 - module.decoder.layers.12.mlp.experts.linear_fc2.weight9 - module.decoder.layers.12.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc1.weight7 module.decoder.layers.13.mlp.experts.linear_fc1.weight5 module.decoder.layers.13.mlp.experts.linear_fc1.weight2 @@ -2225,7 +2216,23 @@ Params for bucket 65 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc2.weight10 module.decoder.layers.12.mlp.experts.linear_fc2.weight8 module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 module.decoder.layers.12.mlp.experts.linear_fc2.weight5 module.decoder.layers.12.mlp.experts.linear_fc2.weight2 module.decoder.layers.12.mlp.experts.linear_fc2.weight3 @@ -2235,32 +2242,25 @@ Params for bucket 66 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight10 module.decoder.layers.12.mlp.experts.linear_fc1.weight8 module.decoder.layers.12.mlp.experts.linear_fc1.weight6 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc1.weight15 - module.decoder.layers.12.mlp.experts.linear_fc1.weight13 - module.decoder.layers.12.mlp.experts.linear_fc1.weight11 - module.decoder.layers.12.mlp.experts.linear_fc1.weight9 - module.decoder.layers.12.mlp.experts.linear_fc1.weight7 Params for bucket 67 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight9 - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 - module.decoder.layers.12.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight8 - module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 module.decoder.layers.11.mlp.experts.linear_fc2.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight10 - module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight15 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight12 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 Params for bucket 68 (40894464 elements, 40894464 padded size): @@ -2281,11 +2281,9 @@ Params for bucket 68 (40894464 elements, 40894464 padded size): Params for bucket 69 (40894464 elements, 40894464 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight12 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 - module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight9 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 @@ -2295,34 +2293,28 @@ Params for bucket 69 (40894464 elements, 40894464 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight11 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight13 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight10 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 Params for bucket 70 (40894464 elements, 40894464 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight14 - module.decoder.layers.10.mlp.experts.linear_fc1.weight10 - module.decoder.layers.10.mlp.experts.linear_fc1.weight12 - module.decoder.layers.10.mlp.experts.linear_fc1.weight8 - module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 - module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight13 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 module.decoder.layers.10.mlp.experts.linear_fc1.weight9 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 Params for bucket 71 (40894464 elements, 40894464 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight13 - module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight10 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight13 - module.decoder.layers.9.mlp.experts.linear_fc2.weight15 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 @@ -2336,28 +2328,29 @@ Params for bucket 71 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight12 - module.decoder.layers.9.mlp.experts.linear_fc1.weight8 module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc1.weight9 - module.decoder.layers.9.mlp.experts.linear_fc1.weight11 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 Params for bucket 73 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight11 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 - module.decoder.layers.8.mlp.experts.linear_fc1.weight14 - module.decoder.layers.8.mlp.experts.linear_fc2.weight8 - module.decoder.layers.8.mlp.experts.linear_fc1.weight11 - module.decoder.layers.8.mlp.experts.linear_fc2.weight13 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 @@ -2372,15 +2365,14 @@ Params for bucket 73 (40894464 elements, 40894464 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight12 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 Params for bucket 74 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight10 - module.decoder.layers.8.mlp.experts.linear_fc1.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight8 - module.decoder.layers.8.mlp.experts.linear_fc1.weight4 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 @@ -2388,15 +2380,15 @@ Params for bucket 74 (40894464 elements, 40894464 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 Params for bucket 75 (40894464 elements, 40894464 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight9 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.7.mlp.experts.linear_fc1.weight12 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight9 - module.decoder.layers.7.mlp.experts.linear_fc2.weight11 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 @@ -2408,51 +2400,51 @@ Params for bucket 75 (40894464 elements, 40894464 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 module.decoder.layers.7.mlp.experts.linear_fc1.weight8 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight14 module.decoder.layers.6.mlp.experts.linear_fc2.weight12 module.decoder.layers.6.mlp.experts.linear_fc2.weight10 module.decoder.layers.6.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc1.weight5 - module.decoder.layers.7.mlp.experts.linear_fc1.weight7 - module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight15 - module.decoder.layers.6.mlp.experts.linear_fc2.weight13 - module.decoder.layers.6.mlp.experts.linear_fc2.weight11 - module.decoder.layers.6.mlp.experts.linear_fc2.weight9 Params for bucket 77 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight15 - module.decoder.layers.6.mlp.experts.linear_fc1.weight13 - module.decoder.layers.6.mlp.experts.linear_fc1.weight11 - module.decoder.layers.6.mlp.experts.linear_fc1.weight9 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight14 module.decoder.layers.6.mlp.experts.linear_fc1.weight12 module.decoder.layers.6.mlp.experts.linear_fc1.weight10 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 78 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight12 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight9 - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight14 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.5.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 @@ -2464,14 +2456,15 @@ Params for bucket 78 (40894464 elements, 40894464 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight15 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 Params for bucket 79 (40894464 elements, 40894464 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight14 - module.decoder.layers.5.mlp.experts.linear_fc1.weight12 - module.decoder.layers.5.mlp.experts.linear_fc1.weight10 - module.decoder.layers.5.mlp.experts.linear_fc1.weight8 - module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight15 @@ -2480,28 +2473,35 @@ Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight9 module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight8 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight10 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight12 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight9 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight14 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 Params for bucket 81 (40894464 elements, 40894464 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight14 module.decoder.layers.4.mlp.experts.linear_fc1.weight12 @@ -2517,28 +2517,34 @@ Params for bucket 81 (40894464 elements, 40894464 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight5 Params for bucket 82 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight8 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight13 - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight11 module.decoder.layers.3.mlp.experts.linear_fc1.weight9 @@ -2546,19 +2552,7 @@ Params for bucket 83 (40894464 elements, 40894464 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight12 - module.decoder.layers.3.mlp.experts.linear_fc1.weight10 - module.decoder.layers.3.mlp.experts.linear_fc1.weight8 - module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 Params for bucket 84 (40894464 elements, 40894464 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight10 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 @@ -2570,10 +2564,16 @@ Params for bucket 84 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight11 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc1.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 Params for bucket 85 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight8 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 @@ -2581,83 +2581,83 @@ Params for bucket 85 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 module.decoder.layers.2.mlp.experts.linear_fc1.weight11 module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight10 module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc2.weight8 - module.decoder.layers.1.mlp.experts.linear_fc1.weight11 - module.decoder.layers.1.mlp.experts.linear_fc2.weight13 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 Params for bucket 87 (40894464 elements, 40894464 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight14 - module.decoder.layers.0.mlp.experts.linear_fc2.weight12 - module.decoder.layers.0.mlp.experts.linear_fc2.weight10 - module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 - module.decoder.layers.0.mlp.experts.linear_fc2.weight13 - module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 Params for bucket 88 (40894464 elements, 40894464 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight8 - module.decoder.layers.0.mlp.experts.linear_fc2.weight6 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 - module.decoder.layers.0.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc1.weight12 - module.decoder.layers.0.mlp.experts.linear_fc1.weight10 - module.decoder.layers.0.mlp.experts.linear_fc1.weight8 module.decoder.layers.0.mlp.experts.linear_fc2.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight11 module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 Params for bucket 89 (25165824 elements, 25165824 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight3 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:00:48 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:46:31 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -2665,7 +2665,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -2678,44 +2678,44 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 15:00:48 +[after dataloaders are built] datetime: 2026-08-26 15:46:31 done with setup ... -training ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (4592.81, 4604.65) - train/valid/test-data-iterators-setup ..........: (75.11, 88.13) + model-and-optimizer-setup ......................: (4598.73, 4609.50) + train/valid/test-data-iterators-setup ..........: (66.40, 82.28) +training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 15:00:48 +[before the start of training step] datetime: 2026-08-26 15:46:31 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -2736,10 +2736,6 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -2756,10 +2752,14 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -2802,1938 +2802,1938 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32190 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31814 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32190 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31814 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=30952 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=31373 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=31373 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=30952 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=31373 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=31373 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32205 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32205 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=30952 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=31723 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=30952 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=31723 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=31723 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=31362 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=31723 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=31362 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=31105 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32137 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=31105 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32137 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32137 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32137 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31814 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31814 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=31033 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32282 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=31033 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=31919 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32205 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32205 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=31919 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32461 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32461 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=40960 expected_recv=30829 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=40960 expected_recv=30829 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32461 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=40960 expected_recv=30829 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32461 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31832 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31832 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31737 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31737 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=31362 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=31362 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=40960 expected_recv=31126 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=40960 expected_recv=31126 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32345 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32345 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=40960 expected_recv=31752 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=40960 expected_recv=31752 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32206 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32206 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=31033 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=31033 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=40960 expected_recv=31752 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=40960 expected_recv=31752 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32345 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=40960 expected_recv=31763 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32345 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=40960 expected_recv=31763 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=40960 expected_recv=30471 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31748 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=40960 expected_recv=30471 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31748 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31832 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=31832 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=31919 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=31919 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31820 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31820 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=40960 expected_recv=31820 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31894 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31894 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31675 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31894 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31675 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=40960 expected_recv=31675 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=40960 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=40960 expected_recv=30473 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32206 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32206 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=40960 expected_recv=30914 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31831 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=40960 expected_recv=30914 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=40960 expected_recv=30914 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31831 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=40960 expected_recv=30471 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=40960 expected_recv=30471 actual_valid_recv=31872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31905 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=40960 expected_recv=31095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=23 calls=1 budget=40960 expected_recv=31285 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31905 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=40960 expected_recv=31095 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=23 calls=1 budget=40960 expected_recv=31285 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=23 calls=1 budget=40960 expected_recv=31285 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=23 calls=1 budget=40960 expected_recv=31285 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=40960 expected_recv=31146 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=40960 expected_recv=31146 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32283 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32283 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=40960 expected_recv=31095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=40960 expected_recv=31095 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31625 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31625 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31943 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31943 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32179 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31905 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32179 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31905 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=25 calls=1 budget=40960 expected_recv=31593 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=25 calls=1 budget=40960 expected_recv=31593 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=40960 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=40960 expected_recv=30473 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31943 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=25 calls=1 budget=40960 expected_recv=31593 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31943 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31831 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=40960 expected_recv=31831 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31852 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32194 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31852 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32194 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31716 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31716 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32365 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32365 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32363 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32367 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32363 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32194 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=29 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32194 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=29 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32365 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32365 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=28 calls=1 budget=40960 expected_recv=31847 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32283 expected_overflow=0 exact_recv_per_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=28 calls=1 budget=40960 expected_recv=31847 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32283 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32280 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32280 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=29 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=29 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31625 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31716 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=24 calls=1 budget=40960 expected_recv=31625 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=26 calls=1 budget=40960 expected_recv=31716 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32359 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=30 calls=1 budget=40960 expected_recv=30602 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32179 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=30 calls=1 budget=40960 expected_recv=30602 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32359 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32363 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32179 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32363 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32359 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32359 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=31 calls=1 budget=40960 expected_recv=31522 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=31 calls=1 budget=40960 expected_recv=31522 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=32 calls=1 budget=40960 expected_recv=31570 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=32 calls=1 budget=40960 expected_recv=31570 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=32 calls=1 budget=40960 expected_recv=31570 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=33 calls=1 budget=40960 expected_recv=31387 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=33 calls=1 budget=40960 expected_recv=31387 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32182 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31945 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31623 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32182 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31945 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31623 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32367 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=34 calls=1 budget=40960 expected_recv=31039 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=34 calls=1 budget=40960 expected_recv=31039 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=28 calls=1 budget=40960 expected_recv=31847 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=28 calls=1 budget=40960 expected_recv=31847 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=36 calls=1 budget=40960 expected_recv=30806 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=37 calls=1 budget=40960 expected_recv=30890 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=36 calls=1 budget=40960 expected_recv=30806 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=37 calls=1 budget=40960 expected_recv=30890 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=36 calls=1 budget=40960 expected_recv=30806 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31945 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=36 calls=1 budget=40960 expected_recv=30806 actual_valid_recv=32128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31945 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32182 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=30 calls=1 budget=40960 expected_recv=30602 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=30 calls=1 budget=40960 expected_recv=30602 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32182 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31975 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31975 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=31 calls=1 budget=40960 expected_recv=31522 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=31 calls=1 budget=40960 expected_recv=31522 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=39 calls=1 budget=40960 expected_recv=31295 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=39 calls=1 budget=40960 expected_recv=31295 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31595 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31595 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31623 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32113 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=35 calls=1 budget=40960 expected_recv=31623 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32113 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=40 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=34 calls=1 budget=40960 expected_recv=31039 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=40 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=34 calls=1 budget=40960 expected_recv=31039 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=41 calls=1 budget=40960 expected_recv=31422 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=41 calls=1 budget=40960 expected_recv=31422 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31595 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=38 calls=1 budget=40960 expected_recv=31595 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32498 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32367 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32498 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31714 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=37 calls=1 budget=40960 expected_recv=30890 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=37 calls=1 budget=40960 expected_recv=30890 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31714 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32184 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32184 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=46 calls=1 budget=40960 expected_recv=30723 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31854 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=46 calls=1 budget=40960 expected_recv=30723 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31854 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31714 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=43 calls=1 budget=40960 expected_recv=31714 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=44 calls=1 budget=40960 expected_recv=31877 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=40 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=44 calls=1 budget=40960 expected_recv=31877 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=46 calls=1 budget=40960 expected_recv=30723 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=46 calls=1 budget=40960 expected_recv=30723 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=40 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=45 calls=1 budget=40960 expected_recv=31680 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32128 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=45 calls=1 budget=40960 expected_recv=31680 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32128 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32498 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=45 calls=1 budget=40960 expected_recv=31680 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32498 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31814 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31814 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32113 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31814 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32113 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=41 calls=1 budget=40960 expected_recv=31422 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=41 calls=1 budget=40960 expected_recv=31422 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31814 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32184 expected_overflow=0 exact_recv_per_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31755 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31755 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32184 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32128 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32128 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31755 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=47 calls=1 budget=40960 expected_recv=31755 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[39091, 37101, 31130, 24876, 39615, 26492, 37070, 26769] avg=32768.0000 max=39615 max_rank=4 min=24876 min_rank=3 imbalance_ratio=1.208954 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33096, 33096, 31814, 33097, 33095, 32190, 33096, 32660] avg=32768.0000 max=33097 max_rank=3 min=31814 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=6518 max_load_reduction_pct=0.164534 moved_to_replicas=20493 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[30041, 33065, 39427, 33916, 21666, 26187, 35859, 41983] avg=32768.0000 max=41983 max_rank=7 min=21666 min_rank=4 imbalance_ratio=1.281219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32805, 33065, 33096, 33096, 31373, 32518, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=8887 max_load_reduction_pct=0.211681 moved_to_replicas=18801 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[8725, 36939, 44516, 30952, 48971, 46743, 20996, 24302] avg=32768.0000 max=48971 max_rank=4 min=8725 min_rank=0 imbalance_ratio=1.494476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33097, 33097, 33096, 30952, 33096, 33097, 33095, 32614] avg=32768.0000 max=33097 max_rank=0 min=30952 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=15874 max_load_reduction_pct=0.324151 moved_to_replicas=44785 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[39217, 44224, 31362, 6848, 25734, 44461, 27940, 42358] avg=32768.0000 max=44461 max_rank=5 min=6848 min_rank=3 imbalance_ratio=1.356842 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[33097, 33096, 31362, 33096, 33096, 33096, 32205, 33096] avg=32768.0000 max=33097 max_rank=0 min=31362 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=11364 max_load_reduction_pct=0.255595 moved_to_replicas=37876 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[66871, 16012, 20304, 64285, 31105, 25603, 20724, 17240] avg=32768.0000 max=66871 max_rank=0 min=16012 min_rank=1 imbalance_ratio=2.040741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 32463, 33096, 31105, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33775 max_load_reduction_pct=0.505077 moved_to_replicas=64964 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[28510, 32804, 42492, 22266, 53367, 11999, 49791, 20915] avg=32768.0000 max=53367 max_rank=4 min=11999 min_rank=5 imbalance_ratio=1.628632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[31723, 32804, 33096, 33096, 33095, 32137, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20270 max_load_reduction_pct=0.379823 moved_to_replicas=46362 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[59625, 22145, 29896, 59464, 14819, 38854, 24059, 13282] avg=32768.0000 max=59625 max_rank=0 min=13282 min_rank=7 imbalance_ratio=1.819611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[33097, 33097, 31033, 33095, 33096, 33095, 32535, 33096] avg=32768.0000 max=33097 max_rank=0 min=31033 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26528 max_load_reduction_pct=0.444914 moved_to_replicas=58655 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[55737, 37442, 6344, 33586, 13557, 19032, 46177, 50269] avg=32768.0000 max=55737 max_rank=0 min=6344 min_rank=2 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33097, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=4 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22640 max_load_reduction_pct=0.406193 moved_to_replicas=57731 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[29881, 27245, 34010, 34584, 38312, 55700, 17595, 24817] avg=32768.0000 max=55700 max_rank=5 min=17595 min_rank=6 imbalance_ratio=1.699829 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32282, 32461, 33096, 33097, 33096, 33096, 33097, 31919] avg=32768.0000 max=33097 max_rank=3 min=31919 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=22603 max_load_reduction_pct=0.405799 moved_to_replicas=30222 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[33277, 39437, 58208, 23866, 38698, 7228, 31832, 29598] avg=32768.0000 max=58208 max_rank=2 min=7228 min_rank=5 imbalance_ratio=1.776367 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 33096, 31737, 31832, 33096] avg=32768.0000 max=33097 max_rank=2 min=31737 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25111 max_load_reduction_pct=0.431401 moved_to_replicas=37236 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[30829, 24985, 45799, 24706, 23193, 23914, 53673, 35045] avg=32768.0000 max=53673 max_rank=6 min=23193 min_rank=4 imbalance_ratio=1.637970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[30829, 33096, 33096, 32739, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30829 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=20577 max_load_reduction_pct=0.383377 moved_to_replicas=35229 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[16418, 26033, 46528, 42399, 34529, 31126, 14105, 51006] avg=32768.0000 max=51006 max_rank=7 min=14105 min_rank=6 imbalance_ratio=1.556580 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[33098, 32440, 33096, 33095, 33097, 31126, 33097, 33095] avg=32768.0000 max=33098 max_rank=0 min=31126 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=17908 max_load_reduction_pct=0.351096 moved_to_replicas=42078 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[40804, 15488, 39032, 26409, 49360, 23770, 38517, 28764] avg=32768.0000 max=49360 max_rank=4 min=15488 min_rank=1 imbalance_ratio=1.506348 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[33097, 31752, 33096, 32345, 33096, 32566, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=16263 max_load_reduction_pct=0.329477 moved_to_replicas=35329 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[25447, 37823, 54541, 36121, 29286, 15649, 36505, 26772] avg=32768.0000 max=54541 max_rank=2 min=15649 min_rank=5 imbalance_ratio=1.664459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32694, 31763, 33097, 32206] avg=32768.0000 max=33097 max_rank=6 min=31763 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=21444 max_load_reduction_pct=0.393172 moved_to_replicas=32606 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[22013, 25074, 18970, 37188, 44079, 50969, 36889, 26962] avg=32768.0000 max=50969 max_rank=5 min=18970 min_rank=2 imbalance_ratio=1.555450 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 33097, 30471] avg=32768.0000 max=33097 max_rank=6 min=30471 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=17872 max_load_reduction_pct=0.350645 moved_to_replicas=36741 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[27427, 21192, 55331, 45582, 42565, 16972, 25050, 28025] avg=32768.0000 max=55331 max_rank=2 min=16972 min_rank=5 imbalance_ratio=1.688568 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[31748, 33096, 33096, 33096, 33096, 31820, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31748 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22235 max_load_reduction_pct=0.401854 moved_to_replicas=44190 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[36456, 10083, 45815, 21089, 37662, 26683, 54688, 29668] avg=32768.0000 max=54688 max_rank=6 min=10083 min_rank=1 imbalance_ratio=1.668945 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[33097, 31675, 33095, 33096, 33095, 31894, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31675 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=21591 max_load_reduction_pct=0.394803 moved_to_replicas=42237 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[42855, 18133, 18356, 29930, 19119, 38224, 71246, 24281] avg=32768.0000 max=71246 max_rank=6 min=18133 min_rank=1 imbalance_ratio=2.174255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[33096, 33095, 33097, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=38149 max_load_reduction_pct=0.535455 moved_to_replicas=53037 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[13276, 37108, 18168, 50766, 18666, 48706, 20340, 55114] avg=32768.0000 max=55114 max_rank=7 min=13276 min_rank=0 imbalance_ratio=1.681946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[33097, 33096, 30473, 33096, 33096, 33096, 33095, 33095] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22017 max_load_reduction_pct=0.399481 moved_to_replicas=59310 active_clone_links=10 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[26925, 30914, 14575, 64312, 39588, 29042, 41354, 15434] avg=32768.0000 max=64312 max_rank=3 min=14575 min_rank=2 imbalance_ratio=1.962646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[33096, 30914, 33096, 33095, 33096, 33096, 33096, 32655] avg=32768.0000 max=33096 max_rank=0 min=30914 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31216 max_load_reduction_pct=0.485384 moved_to_replicas=45966 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[16969, 37880, 42774, 50532, 12798, 58173, 18596, 24422] avg=32768.0000 max=58173 max_rank=5 min=12798 min_rank=4 imbalance_ratio=1.775299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[32928, 33096, 33096, 33096, 33097, 33095, 31905, 31831] avg=32768.0000 max=33097 max_rank=4 min=31831 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25076 max_load_reduction_pct=0.431059 moved_to_replicas=56975 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[28117, 18813, 11076, 39018, 26518, 55851, 32813, 49938] avg=32768.0000 max=55851 max_rank=5 min=11076 min_rank=2 imbalance_ratio=1.704437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[33095, 33096, 32756, 33096, 31095, 33096, 32813, 33097] avg=32768.0000 max=33097 max_rank=7 min=31095 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22754 max_load_reduction_pct=0.407405 moved_to_replicas=45519 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[28944, 18259, 41405, 40623, 47315, 26988, 30345, 28265] avg=32768.0000 max=47315 max_rank=4 min=18259 min_rank=1 imbalance_ratio=1.443939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[32422, 33096, 33097, 33096, 33096, 31146, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31146 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=14218 max_load_reduction_pct=0.300497 moved_to_replicas=30055 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[26664, 43965, 19050, 48411, 54679, 9720, 37693, 21962] avg=32768.0000 max=54679 max_rank=4 min=9720 min_rank=5 imbalance_ratio=1.668671 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[31285, 33096, 33096, 33096, 33096, 33096, 33096, 32283] avg=32768.0000 max=33096 max_rank=1 min=31285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=21583 max_load_reduction_pct=0.394722 moved_to_replicas=52364 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=25 tokens_per_ep_rank=[22858, 39482, 36718, 38671, 25557, 20373, 49280, 29205] avg=32768.0000 max=49280 max_rank=6 min=20373 min_rank=5 imbalance_ratio=1.503906 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31943, 33096, 33096, 31625] avg=32768.0000 max=33096 max_rank=0 min=31625 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16184 max_load_reduction_pct=0.328409 moved_to_replicas=31767 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=26 tokens_per_ep_rank=[40154, 43567, 34649, 38482, 20868, 31593, 21680, 31151] avg=32768.0000 max=43567 max_rank=1 min=20868 min_rank=4 imbalance_ratio=1.329559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=26 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 32892, 31593, 33095, 32179] avg=32768.0000 max=33097 max_rank=0 min=31593 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10470 max_load_reduction_pct=0.240320 moved_to_replicas=24468 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=27 tokens_per_ep_rank=[36801, 23180, 33590, 46279, 47820, 25524, 26105, 22845] avg=32768.0000 max=47820 max_rank=4 min=22845 min_rank=7 imbalance_ratio=1.459351 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=27 tokens_per_ep_rank=[33097, 31852, 33096, 33095, 33096, 33097, 31716, 33095] avg=32768.0000 max=33097 max_rank=0 min=31716 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=14723 max_load_reduction_pct=0.307884 moved_to_replicas=32106 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=28 tokens_per_ep_rank=[34256, 32837, 28365, 35938, 24033, 40616, 32363, 33736] avg=32768.0000 max=40616 max_rank=5 min=24033 min_rank=4 imbalance_ratio=1.239502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=28 tokens_per_ep_rank=[33096, 32837, 32367, 33096, 32194, 33095, 32363, 33096] avg=32768.0000 max=33096 max_rank=0 min=32194 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=7520 max_load_reduction_pct=0.185149 moved_to_replicas=12162 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=29 tokens_per_ep_rank=[35747, 44827, 22049, 35559, 24422, 27611, 57685, 14244] avg=32768.0000 max=57685 max_rank=6 min=14244 min_rank=7 imbalance_ratio=1.760406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=29 tokens_per_ep_rank=[33096, 33096, 31847, 33096, 32365, 33096, 33097, 32451] avg=32768.0000 max=33097 max_rank=6 min=31847 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24588 max_load_reduction_pct=0.426246 moved_to_replicas=41434 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=30 tokens_per_ep_rank=[49686, 38730, 18752, 15509, 44206, 28231, 23952, 43078] avg=32768.0000 max=49686 max_rank=0 min=15509 min_rank=3 imbalance_ratio=1.516296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 30472, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=16590 max_load_reduction_pct=0.333897 moved_to_replicas=43316 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=31 tokens_per_ep_rank=[43595, 42429, 16971, 35071, 14948, 27452, 43067, 38611] avg=32768.0000 max=43595 max_rank=0 min=14948 min_rank=4 imbalance_ratio=1.330414 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=31 tokens_per_ep_rank=[33096, 33096, 30602, 33096, 33096, 32967, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=30602 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=10499 max_load_reduction_pct=0.240830 moved_to_replicas=37293 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=32 tokens_per_ep_rank=[16696, 47280, 23709, 30333, 35997, 35122, 43124, 29883] avg=32768.0000 max=47280 max_rank=1 min=16696 min_rank=0 imbalance_ratio=1.442871 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=32 tokens_per_ep_rank=[33095, 33096, 31522, 32359, 33095, 33096, 33096, 32785] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14184 max_load_reduction_pct=0.300000 moved_to_replicas=29139 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=33 tokens_per_ep_rank=[12073, 24645, 44510, 41888, 21813, 35426, 37691, 44098] avg=32768.0000 max=44510 max_rank=2 min=12073 min_rank=0 imbalance_ratio=1.358337 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=33 tokens_per_ep_rank=[32280, 31570, 33095, 33096, 32815, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31570 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=11414 max_load_reduction_pct=0.256437 moved_to_replicas=38133 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=34 tokens_per_ep_rank=[53287, 25793, 46999, 41259, 25667, 19498, 15171, 34470] avg=32768.0000 max=53287 max_rank=0 min=15171 min_rank=6 imbalance_ratio=1.626190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=34 tokens_per_ep_rank=[33095, 31387, 33096, 33096, 33095, 33097, 32182, 33096] avg=32768.0000 max=33097 max_rank=5 min=31387 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20190 max_load_reduction_pct=0.378892 moved_to_replicas=43631 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=35 tokens_per_ep_rank=[35776, 43013, 24172, 48067, 23166, 25005, 32797, 30148] avg=32768.0000 max=48067 max_rank=3 min=23166 min_rank=4 imbalance_ratio=1.466888 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=35 tokens_per_ep_rank=[33096, 33096, 31039, 33096, 33096, 33096, 32797, 32828] avg=32768.0000 max=33096 max_rank=0 min=31039 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14971 max_load_reduction_pct=0.311461 moved_to_replicas=27568 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=36 tokens_per_ep_rank=[18974, 29297, 59483, 39886, 21214, 39048, 31034, 23208] avg=32768.0000 max=59483 max_rank=2 min=18974 min_rank=0 imbalance_ratio=1.815277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=36 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31945, 33096, 33096, 31623] avg=32768.0000 max=33096 max_rank=0 min=31623 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26387 max_load_reduction_pct=0.443606 moved_to_replicas=39129 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=37 tokens_per_ep_rank=[45920, 43848, 57460, 17116, 25997, 37996, 20044, 13763] avg=32768.0000 max=57460 max_rank=2 min=13763 min_rank=7 imbalance_ratio=1.753540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 30806, 33096, 33096, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24364 max_load_reduction_pct=0.424017 moved_to_replicas=52840 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=38 tokens_per_ep_rank=[32679, 27548, 41292, 46188, 37602, 18005, 39198, 19632] avg=32768.0000 max=46188 max_rank=3 min=18005 min_rank=5 imbalance_ratio=1.409546 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=38 tokens_per_ep_rank=[32679, 33096, 33096, 33096, 33096, 33095, 33096, 30890] avg=32768.0000 max=33096 max_rank=1 min=30890 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=13092 max_load_reduction_pct=0.283450 moved_to_replicas=31896 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=39 tokens_per_ep_rank=[33535, 37665, 26000, 29595, 47073, 20192, 24500, 43584] avg=32768.0000 max=47073 max_rank=4 min=20192 min_rank=5 imbalance_ratio=1.436554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=39 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33095, 31975, 31595, 33096] avg=32768.0000 max=33096 max_rank=1 min=31595 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=13977 max_load_reduction_pct=0.296922 moved_to_replicas=29473 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=40 tokens_per_ep_rank=[25581, 42147, 39490, 42486, 23330, 31295, 26992, 30823] avg=32768.0000 max=42486 max_rank=3 min=23330 min_rank=4 imbalance_ratio=1.296570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=40 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32721, 31295, 33095, 32650] avg=32768.0000 max=33096 max_rank=0 min=31295 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=9390 max_load_reduction_pct=0.221014 moved_to_replicas=24835 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=41 tokens_per_ep_rank=[34069, 21799, 22225, 14214, 50827, 45291, 22274, 51445] avg=32768.0000 max=51445 max_rank=7 min=14214 min_rank=3 imbalance_ratio=1.569977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=41 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18348 max_load_reduction_pct=0.356653 moved_to_replicas=49248 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=42 tokens_per_ep_rank=[14173, 34548, 31422, 49748, 35414, 39323, 26271, 31245] avg=32768.0000 max=49748 max_rank=3 min=14173 min_rank=0 imbalance_ratio=1.518188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=42 tokens_per_ep_rank=[33095, 33096, 31422, 33096, 33097, 33096, 32498, 32744] avg=32768.0000 max=33097 max_rank=4 min=31422 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16651 max_load_reduction_pct=0.334707 moved_to_replicas=26649 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=43 tokens_per_ep_rank=[16701, 23142, 32184, 48540, 38681, 39007, 34829, 29060] avg=32768.0000 max=48540 max_rank=3 min=16701 min_rank=0 imbalance_ratio=1.481323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=43 tokens_per_ep_rank=[33096, 32367, 32184, 33096, 33096, 33096, 33096, 32113] avg=32768.0000 max=33096 max_rank=0 min=32113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=15444 max_load_reduction_pct=0.318171 moved_to_replicas=28673 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=44 tokens_per_ep_rank=[20764, 25583, 33876, 29684, 37693, 34120, 61817, 18607] avg=32768.0000 max=61817 max_rank=6 min=18607 min_rank=7 imbalance_ratio=1.886505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=44 tokens_per_ep_rank=[33094, 31854, 33096, 31714, 33097, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=4 min=31714 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28720 max_load_reduction_pct=0.464597 moved_to_replicas=35122 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=45 tokens_per_ep_rank=[37516, 44220, 14017, 40881, 25452, 19931, 41485, 38642] avg=32768.0000 max=44220 max_rank=1 min=14017 min_rank=2 imbalance_ratio=1.349487 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=48 total_expected_recv=1571071 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341760 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=45 tokens_per_ep_rank=[33095, 33096, 32128, 33096, 32660, 31877, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31877 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11124 max_load_reduction_pct=0.251560 moved_to_replicas=37264 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=46 tokens_per_ep_rank=[34209, 16217, 48167, 35196, 35120, 29655, 29144, 34436] avg=32768.0000 max=48167 max_rank=2 min=16217 min_rank=1 imbalance_ratio=1.469940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=46 tokens_per_ep_rank=[33096, 32401, 33096, 33096, 33095, 31680, 32584, 33096] avg=32768.0000 max=33096 max_rank=0 min=31680 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15071 max_load_reduction_pct=0.312891 moved_to_replicas=21648 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=47 tokens_per_ep_rank=[26528, 25758, 15996, 39305, 35383, 43736, 42210, 33228] avg=32768.0000 max=43736 max_rank=5 min=15996 min_rank=2 imbalance_ratio=1.334717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=47 tokens_per_ep_rank=[30723, 33096, 32845, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30723 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=10639 max_load_reduction_pct=0.243255 moved_to_replicas=28382 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=48 tokens_per_ep_rank=[36035, 35720, 23924, 31416, 47998, 33346, 38802, 14903] avg=32768.0000 max=47998 max_rank=4 min=14903 min_rank=7 imbalance_ratio=1.464783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=48 tokens_per_ep_rank=[33095, 33096, 31755, 31814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31755 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14902 max_load_reduction_pct=0.310471 moved_to_replicas=26421 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=48 total_expected_recv=1576257 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=337152 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=48 total_expected_recv=1569356 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344960 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=48 total_expected_recv=1574413 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=336768 peak_demand=33097 required_capacity=1.010040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=48 total_expected_recv=1577256 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=335744 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=48 total_expected_recv=1569356 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344960 peak_demand=33097 required_capacity=1.010040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=48 total_expected_recv=1574642 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=337792 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=48 total_expected_recv=1575856 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=339712 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=48 total_expected_recv=1574413 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=336768 peak_demand=33097 required_capacity=1.010040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=48 total_expected_recv=1564061 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=348544 peak_demand=33097 required_capacity=1.010040 - [2026-08-26 15:01:18] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 29969.1 | throughput per GPU (TFLOP/s/GPU): 3.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095286E+01 | loss scale: 1.0 | grad norm: 54.363 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -Number of parameters in transformer block in billions: 29.90 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=48 total_expected_recv=1575856 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=339712 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=48 total_expected_recv=1571071 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341760 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:46:58] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 26592.4 | throughput per GPU (TFLOP/s/GPU): 3.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095286E+01 | loss scale: 1.0 | grad norm: 54.363 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 |Number of parameters in transformer block in billions: 29.90 + Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 30.10 Number of parameters in most loaded shard in billions: 30.1034 Theoretical memory footprints: weight and optimizer=215316.59 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 72351.18896484375 | max allocated: 79924.4072265625 | reserved: 80338.0 | max reserved: 80338.0 +[Rank 0] (after 1 iterations) memory (MB) | allocated: 72350.68896484375 | max allocated: 79925.3603515625 | reserved: 80330.0 | max reserved: 80330.0 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -4750,21235 +4750,21243 @@ Theoretical memory footprints: weight and optimizer=215316.59 MB warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=31432 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=31432 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=31432 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=31576 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=31576 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=30947 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=30947 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32318 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=31252 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32318 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=31252 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32318 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32318 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=31252 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=31252 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32322 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=31115 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=31245 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32322 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=31115 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=31245 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33022 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=31115 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33022 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=31115 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31784 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31783 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31784 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31784 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31784 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=31783 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=30922 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=30922 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=30922 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=30880 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=30922 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=30880 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=31125 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=30880 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=30880 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=31125 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=40960 expected_recv=30714 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=40960 expected_recv=30714 actual_valid_recv=31872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34816 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=31125 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=31125 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32302 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32302 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34816 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=40960 expected_recv=30714 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=40960 expected_recv=30714 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=40960 expected_recv=31265 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=40960 expected_recv=31415 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=40960 expected_recv=31265 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=40960 expected_recv=31415 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32302 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32302 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=40960 expected_recv=31265 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=40960 expected_recv=31265 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=40960 expected_recv=31415 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=40960 expected_recv=31415 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32030 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32314 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=40960 expected_recv=31542 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=40960 expected_recv=31845 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32314 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32030 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=40960 expected_recv=31845 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=40960 expected_recv=31542 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32030 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32030 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=40960 expected_recv=30652 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=40960 expected_recv=31845 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32314 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=40960 expected_recv=30652 actual_valid_recv=31872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32314 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=40960 expected_recv=31845 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=40960 expected_recv=31542 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=40960 expected_recv=31542 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=40960 expected_recv=30652 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32267 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32191 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=40960 expected_recv=30652 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32267 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32191 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32267 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32267 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=40960 expected_recv=31301 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=40960 expected_recv=31301 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32265 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32265 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32265 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=40960 expected_recv=31732 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32191 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=40960 expected_recv=31732 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31614 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32209 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32191 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32316 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31614 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32209 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32316 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=40960 expected_recv=31732 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=40960 expected_recv=31732 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31954 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31954 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31614 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31614 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32316 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32316 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31954 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=40960 expected_recv=31954 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31896 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31896 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=40960 expected_recv=30924 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=40960 expected_recv=30924 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=40960 expected_recv=30607 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=40960 expected_recv=30607 actual_valid_recv=31360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31695 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33074 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31695 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33074 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=40960 expected_recv=30924 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=40960 expected_recv=30607 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31896 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=23 calls=1 budget=40960 expected_recv=31668 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31896 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=23 calls=1 budget=40960 expected_recv=31668 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32260 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32260 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=24 calls=1 budget=40960 expected_recv=30783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=24 calls=1 budget=40960 expected_recv=30783 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31695 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=40960 expected_recv=31695 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32260 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32260 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32333 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32084 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33074 expected_overflow=0 exact_recv_per_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=23 calls=1 budget=40960 expected_recv=31668 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32333 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=40960 expected_recv=33074 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=23 calls=1 budget=40960 expected_recv=31668 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32244 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32244 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32244 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=26 calls=1 budget=40960 expected_recv=31521 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32047 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=24 calls=1 budget=40960 expected_recv=30783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32047 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32333 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=26 calls=1 budget=40960 expected_recv=31521 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=24 calls=1 budget=40960 expected_recv=30783 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32333 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33043 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33043 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=27 calls=1 budget=40960 expected_recv=31527 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=26 calls=1 budget=40960 expected_recv=31521 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32119 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=27 calls=1 budget=40960 expected_recv=31527 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=26 calls=1 budget=40960 expected_recv=31521 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32047 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32119 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32047 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32119 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32102 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32102 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32084 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=28 calls=1 budget=40960 expected_recv=31788 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=28 calls=1 budget=40960 expected_recv=31788 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=28 calls=1 budget=40960 expected_recv=31788 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32284 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32284 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33043 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33043 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=29 calls=1 budget=40960 expected_recv=31940 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=29 calls=1 budget=40960 expected_recv=31940 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=30 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32102 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=30 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=27 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32102 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32259 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=29 calls=1 budget=40960 expected_recv=31940 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=29 calls=1 budget=40960 expected_recv=31940 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=31 calls=1 budget=40960 expected_recv=31685 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32259 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=31 calls=1 budget=40960 expected_recv=31685 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31945 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31945 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32284 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32284 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31621 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32178 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31621 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32178 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33071 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33071 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=33 calls=1 budget=40960 expected_recv=31415 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=33 calls=1 budget=40960 expected_recv=31415 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=34 calls=1 budget=40960 expected_recv=31009 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=34 calls=1 budget=40960 expected_recv=31009 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32259 expected_overflow=0 exact_recv_per_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32259 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31930 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31930 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33071 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31638 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31621 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33071 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31638 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31621 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32178 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31945 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32178 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=32 calls=1 budget=40960 expected_recv=31945 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32009 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32009 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=34 calls=1 budget=40960 expected_recv=31009 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=33 calls=1 budget=40960 expected_recv=31415 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=34 calls=1 budget=40960 expected_recv=31009 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=36 calls=1 budget=40960 expected_recv=31560 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=33 calls=1 budget=40960 expected_recv=31415 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=36 calls=1 budget=40960 expected_recv=31560 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32059 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32059 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31930 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=37 calls=1 budget=40960 expected_recv=31508 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=35 calls=1 budget=40960 expected_recv=31930 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=37 calls=1 budget=40960 expected_recv=31508 actual_valid_recv=32384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=36 calls=1 budget=40960 expected_recv=31560 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=36 calls=1 budget=40960 expected_recv=31560 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31696 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31696 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=37 calls=1 budget=40960 expected_recv=31508 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=37 calls=1 budget=40960 expected_recv=31508 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31872 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32009 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31872 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32009 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31872 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=38 calls=1 budget=40960 expected_recv=31872 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=39 calls=1 budget=40960 expected_recv=30516 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=39 calls=1 budget=40960 expected_recv=30516 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33050 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31913 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33050 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31913 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=39 calls=1 budget=40960 expected_recv=30516 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=39 calls=1 budget=40960 expected_recv=30516 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31655 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32059 expected_overflow=0 exact_recv_per_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32059 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31655 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=41 calls=1 budget=40960 expected_recv=30551 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=42 calls=1 budget=40960 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=41 calls=1 budget=40960 expected_recv=30551 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33050 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=42 calls=1 budget=40960 expected_recv=30473 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33050 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31663 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31663 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31655 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31906 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31906 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31655 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31913 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=41 calls=1 budget=40960 expected_recv=30551 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=40 calls=1 budget=40960 expected_recv=31913 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=41 calls=1 budget=40960 expected_recv=30551 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=44 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=44 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31906 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31850 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31718 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31906 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31850 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31718 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31663 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=43 calls=1 budget=40960 expected_recv=31663 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=46 calls=1 budget=40960 expected_recv=30605 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=46 calls=1 budget=40960 expected_recv=30605 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31718 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31718 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=46 calls=1 budget=40960 expected_recv=30605 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31947 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31947 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31850 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=45 calls=1 budget=40960 expected_recv=31850 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31621 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31621 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31621 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31621 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31947 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=47 calls=1 budget=40960 expected_recv=31947 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[39420, 37858, 30475, 24939, 38625, 26806, 36080, 27941] avg=32768.0000 max=39420 max_rank=0 min=24939 min_rank=3 imbalance_ratio=1.203003 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[33096, 33096, 31432, 33095, 33096, 32552, 33095, 32682] avg=32768.0000 max=33096 max_rank=0 min=31432 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=6324 max_load_reduction_pct=0.160426 moved_to_replicas=19599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[29619, 32500, 36761, 37240, 18965, 27432, 34319, 45308] avg=32768.0000 max=45308 max_rank=7 min=18965 min_rank=4 imbalance_ratio=1.382690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32500, 33096, 33096, 32589, 31576, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31576 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=21244 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[8575, 39723, 41787, 30292, 52651, 44935, 19108, 25073] avg=32768.0000 max=52651 max_rank=4 min=8575 min_rank=0 imbalance_ratio=1.606781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33097, 33097, 32621, 33095, 33096, 30947, 33095] avg=32768.0000 max=33097 max_rank=1 min=30947 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19554 max_load_reduction_pct=0.371389 moved_to_replicas=46712 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[38396, 43671, 37711, 7545, 22361, 40339, 33683, 38438] avg=32768.0000 max=43671 max_rank=1 min=7545 min_rank=3 imbalance_ratio=1.332733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33096, 33095, 33096, 31252, 32318, 33095, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31252 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=10575 max_load_reduction_pct=0.242152 moved_to_replicas=33662 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[64279, 20926, 27015, 69901, 25301, 24284, 13523, 16915] avg=32768.0000 max=69901 max_rank=3 min=13523 min_rank=6 imbalance_ratio=2.133209 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33096, 31245, 33096, 33097, 32322, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31245 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=36804 max_load_reduction_pct=0.526516 moved_to_replicas=67988 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[24290, 29581, 34630, 17595, 54819, 17893, 42002, 41334] avg=32768.0000 max=54819 max_rank=4 min=17595 min_rank=3 imbalance_ratio=1.672943 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32528, 31115, 33096, 33096, 33096, 33022, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31115 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21723 max_load_reduction_pct=0.396268 moved_to_replicas=40401 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[64781, 2490, 41144, 59572, 35760, 22376, 12283, 23738] avg=32768.0000 max=64781 max_rank=0 min=2490 min_rank=1 imbalance_ratio=1.976959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 33098, 33096, 33095, 33096, 31783, 31784] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31683 max_load_reduction_pct=0.489079 moved_to_replicas=68873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[48037, 34976, 13805, 24072, 14378, 19443, 38333, 69100] avg=32768.0000 max=69100 max_rank=7 min=13805 min_rank=2 imbalance_ratio=2.108765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 30922, 33097, 32647, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30922 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=36003 max_load_reduction_pct=0.521027 moved_to_replicas=58062 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[17863, 22034, 54236, 35596, 28418, 60631, 22980, 20386] avg=32768.0000 max=60631 max_rank=5 min=17863 min_rank=0 imbalance_ratio=1.850311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[33095, 33096, 33096, 33097, 30880, 33096, 33095, 32689] avg=32768.0000 max=33097 max_rank=3 min=30880 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=27534 max_load_reduction_pct=0.454124 moved_to_replicas=51175 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[36798, 32442, 55468, 18302, 25435, 14158, 38440, 41101] avg=32768.0000 max=55468 max_rank=2 min=14158 min_rank=5 imbalance_ratio=1.692749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33096, 32442, 33096, 33094, 31125, 33097, 33097, 33097] avg=32768.0000 max=33097 max_rank=5 min=31125 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=22371 max_load_reduction_pct=0.403314 moved_to_replicas=39423 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[23113, 26718, 43966, 34134, 32853, 44699, 34289, 22372] avg=32768.0000 max=44699 max_rank=5 min=22372 min_rank=7 imbalance_ratio=1.364105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[33097, 30714, 33096, 33096, 32853, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=30714 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=11602 max_load_reduction_pct=0.259558 moved_to_replicas=24704 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[7270, 33267, 14775, 53949, 18878, 46989, 24985, 62031] avg=32768.0000 max=62031 max_rank=7 min=7270 min_rank=0 imbalance_ratio=1.893036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[32302, 33097, 33097, 33096, 31265, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31265 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=28934 max_load_reduction_pct=0.466444 moved_to_replicas=63852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[25268, 30043, 46828, 15441, 32634, 35668, 26159, 50103] avg=32768.0000 max=50103 max_rank=7 min=15441 min_rank=3 imbalance_ratio=1.529022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[31415, 32615, 33095, 33097, 32634, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31415 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17006 max_load_reduction_pct=0.339421 moved_to_replicas=33311 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[23974, 34008, 57980, 15776, 26588, 25855, 47656, 30307] avg=32768.0000 max=57980 max_rank=2 min=15776 min_rank=3 imbalance_ratio=1.769409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[32791, 33096, 33094, 31845, 33096, 33096, 33096, 32030] avg=32768.0000 max=33096 max_rank=1 min=31845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=24884 max_load_reduction_pct=0.429182 moved_to_replicas=40356 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[30048, 13315, 27703, 44517, 34421, 62006, 35041, 15093] avg=32768.0000 max=62006 max_rank=5 min=13315 min_rank=1 imbalance_ratio=1.892273 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[32314, 31542, 32808, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28910 max_load_reduction_pct=0.466245 moved_to_replicas=43601 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[37891, 11183, 58934, 56402, 43802, 17459, 23503, 12970] avg=32768.0000 max=58934 max_rank=2 min=11183 min_rank=1 imbalance_ratio=1.798523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[33097, 30652, 33095, 33096, 33096, 33096, 32917, 33095] avg=32768.0000 max=33097 max_rank=0 min=30652 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=25837 max_load_reduction_pct=0.438406 moved_to_replicas=64645 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[25310, 21189, 31301, 18910, 25047, 36607, 61083, 42697] avg=32768.0000 max=61083 max_rank=6 min=18910 min_rank=3 imbalance_ratio=1.864105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[33095, 33095, 31301, 33097, 32267, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31301 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=27986 max_load_reduction_pct=0.458163 moved_to_replicas=41099 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[64641, 17018, 7097, 43728, 21291, 28233, 63459, 16677] avg=32768.0000 max=64641 max_rank=0 min=7097 min_rank=2 imbalance_ratio=1.972687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[33094, 32191, 32209, 33096, 33096, 32265, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32191 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31544 max_load_reduction_pct=0.487988 moved_to_replicas=72540 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[24872, 48056, 26638, 32316, 28938, 33615, 13156, 54553] avg=32768.0000 max=54553 max_rank=7 min=13156 min_rank=6 imbalance_ratio=1.664825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[33096, 33096, 32616, 32316, 31732, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31732 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21457 max_load_reduction_pct=0.393324 moved_to_replicas=36936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[30611, 10612, 19283, 65892, 33854, 22449, 60303, 19140] avg=32768.0000 max=65892 max_rank=3 min=10612 min_rank=1 imbalance_ratio=2.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[31614, 33096, 33096, 33096, 33096, 33097, 33095, 31954] avg=32768.0000 max=33097 max_rank=5 min=31614 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32795 max_load_reduction_pct=0.497708 moved_to_replicas=60761 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[33115, 42797, 16534, 52986, 14854, 51725, 15471, 34662] avg=32768.0000 max=52986 max_rank=3 min=14854 min_rank=4 imbalance_ratio=1.617004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[33096, 33096, 30924, 33096, 32644, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=30924 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=19889 max_load_reduction_pct=0.375363 moved_to_replicas=49805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[52684, 48511, 8830, 34075, 24937, 57228, 9281, 26598] avg=32768.0000 max=57228 max_rank=5 min=8830 min_rank=2 imbalance_ratio=1.746460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[33096, 33096, 32961, 33095, 33096, 33097, 30607, 33096] avg=32768.0000 max=33097 max_rank=5 min=30607 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24131 max_load_reduction_pct=0.421664 moved_to_replicas=60114 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[25739, 12100, 44596, 33074, 49869, 35417, 33797, 27552] avg=32768.0000 max=49869 max_rank=4 min=12100 min_rank=1 imbalance_ratio=1.521881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[33096, 31896, 33096, 33074, 33095, 33096, 33096, 31695] avg=32768.0000 max=33096 max_rank=0 min=31695 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16773 max_load_reduction_pct=0.336341 moved_to_replicas=31295 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[22988, 51572, 6718, 71926, 24874, 24295, 34326, 25445] avg=32768.0000 max=71926 max_rank=3 min=6718 min_rank=2 imbalance_ratio=2.195007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[33040, 33097, 32791, 33095, 33096, 32260, 33097, 31668] avg=32768.0000 max=33097 max_rank=1 min=31668 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=38829 max_load_reduction_pct=0.539847 moved_to_replicas=58536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=25 tokens_per_ep_rank=[41813, 36246, 41316, 45144, 24068, 13137, 42938, 17482] avg=32768.0000 max=45144 max_rank=3 min=13137 min_rank=5 imbalance_ratio=1.377686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=25 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 32785, 33095, 33096, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12047 max_load_reduction_pct=0.266857 moved_to_replicas=41977 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=26 tokens_per_ep_rank=[33425, 31679, 33751, 27363, 18650, 37490, 39669, 40117] avg=32768.0000 max=40117 max_rank=7 min=18650 min_rank=4 imbalance_ratio=1.224274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=26 tokens_per_ep_rank=[33097, 32333, 33097, 32084, 32244, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=32084 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=7020 max_load_reduction_pct=0.174988 moved_to_replicas=18972 active_clone_links=5 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=27 tokens_per_ep_rank=[38305, 21302, 23639, 47897, 53915, 33962, 21998, 21126] avg=32768.0000 max=53915 max_rank=4 min=21126 min_rank=7 imbalance_ratio=1.645355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=27 tokens_per_ep_rank=[33096, 31521, 33095, 33096, 33096, 33096, 32047, 33097] avg=32768.0000 max=33097 max_rank=7 min=31521 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20818 max_load_reduction_pct=0.386126 moved_to_replicas=41695 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=28 tokens_per_ep_rank=[27453, 44892, 42784, 30393, 10635, 35775, 37169, 33043] avg=32768.0000 max=44892 max_rank=1 min=10635 min_rank=4 imbalance_ratio=1.369995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=28 tokens_per_ep_rank=[31527, 33096, 33096, 33072, 32119, 33096, 33095, 33043] avg=32768.0000 max=33096 max_rank=1 min=31527 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=11796 max_load_reduction_pct=0.262764 moved_to_replicas=28236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=29 tokens_per_ep_rank=[29268, 56509, 14462, 23942, 31679, 25537, 63767, 16980] avg=32768.0000 max=63767 max_rank=6 min=14462 min_rank=2 imbalance_ratio=1.946014 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=29 tokens_per_ep_rank=[33096, 33096, 32774, 33096, 31788, 33096, 33096, 32102] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30671 max_load_reduction_pct=0.480985 moved_to_replicas=54084 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=30 tokens_per_ep_rank=[62166, 26450, 30625, 13771, 28341, 31708, 22539, 46544] avg=32768.0000 max=62166 max_rank=0 min=13771 min_rank=3 imbalance_ratio=1.897156 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=30 tokens_per_ep_rank=[33096, 33096, 33096, 32284, 32440, 31940, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31940 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29070 max_load_reduction_pct=0.467619 moved_to_replicas=42518 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=31 tokens_per_ep_rank=[27621, 63010, 16034, 43401, 7086, 28332, 36774, 39886] avg=32768.0000 max=63010 max_rank=1 min=7086 min_rank=4 imbalance_ratio=1.922913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=31 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=29913 max_load_reduction_pct=0.474734 moved_to_replicas=50687 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=32 tokens_per_ep_rank=[29655, 29487, 32720, 32259, 43047, 45395, 33935, 15646] avg=32768.0000 max=45395 max_rank=5 min=15646 min_rank=7 imbalance_ratio=1.385345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=32 tokens_per_ep_rank=[31685, 33096, 32720, 32259, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31685 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12298 max_load_reduction_pct=0.270911 moved_to_replicas=23089 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=33 tokens_per_ep_rank=[19017, 40021, 20254, 28177, 34156, 36413, 40508, 43598] avg=32768.0000 max=43598 max_rank=7 min=19017 min_rank=0 imbalance_ratio=1.330505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=33 tokens_per_ep_rank=[33096, 33097, 31621, 31945, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31621 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=10501 max_load_reduction_pct=0.240860 moved_to_replicas=29216 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=34 tokens_per_ep_rank=[63867, 22570, 31415, 35441, 25995, 34117, 22683, 26056] avg=32768.0000 max=63867 max_rank=0 min=22570 min_rank=1 imbalance_ratio=1.949066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=34 tokens_per_ep_rank=[33096, 33096, 31415, 33096, 33096, 33096, 33071, 32178] avg=32768.0000 max=33096 max_rank=0 min=31415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30771 max_load_reduction_pct=0.481798 moved_to_replicas=34137 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=35 tokens_per_ep_rank=[34775, 27486, 16104, 58017, 30879, 12875, 28813, 53195] avg=32768.0000 max=58017 max_rank=3 min=12875 min_rank=5 imbalance_ratio=1.770538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=35 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32558, 33097, 31009, 33096] avg=32768.0000 max=33097 max_rank=5 min=31009 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=24920 max_load_reduction_pct=0.429529 moved_to_replicas=46699 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=36 tokens_per_ep_rank=[30282, 45365, 49075, 42252, 24763, 17316, 21161, 31930] avg=32768.0000 max=49075 max_rank=2 min=17316 min_rank=5 imbalance_ratio=1.497650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=36 tokens_per_ep_rank=[31638, 33096, 33096, 33096, 33096, 33096, 33096, 31930] avg=32768.0000 max=33096 max_rank=1 min=31638 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15979 max_load_reduction_pct=0.325604 moved_to_replicas=37404 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=37 tokens_per_ep_rank=[48897, 29365, 42396, 30764, 39451, 25706, 25451, 20114] avg=32768.0000 max=48897 max_rank=0 min=20114 min_rank=7 imbalance_ratio=1.492218 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=37 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33095, 31560, 32009] avg=32768.0000 max=33096 max_rank=0 min=31560 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15801 max_load_reduction_pct=0.323149 moved_to_replicas=31456 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=38 tokens_per_ep_rank=[45693, 23613, 24464, 31171, 38285, 11270, 53664, 33984] avg=32768.0000 max=53664 max_rank=6 min=11270 min_rank=5 imbalance_ratio=1.637695 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=38 tokens_per_ep_rank=[33097, 31508, 33095, 32059, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31508 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20566 max_load_reduction_pct=0.383236 moved_to_replicas=39242 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=39 tokens_per_ep_rank=[22022, 23798, 36466, 11734, 58609, 23052, 44230, 42233] avg=32768.0000 max=58609 max_rank=4 min=11734 min_rank=3 imbalance_ratio=1.788605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=39 tokens_per_ep_rank=[31696, 31872, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31696 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25513 max_load_reduction_pct=0.435309 moved_to_replicas=49154 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=40 tokens_per_ep_rank=[27202, 23266, 33050, 63811, 39078, 34174, 25650, 15913] avg=32768.0000 max=63811 max_rank=3 min=15913 min_rank=7 imbalance_ratio=1.947357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=40 tokens_per_ep_rank=[33096, 30516, 33050, 33097, 33096, 33096, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=30516 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30713 max_load_reduction_pct=0.481312 moved_to_replicas=37775 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=41 tokens_per_ep_rank=[51800, 23002, 11919, 22935, 37686, 33851, 25370, 55581] avg=32768.0000 max=55581 max_rank=7 min=11919 min_rank=2 imbalance_ratio=1.696198 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=41 tokens_per_ep_rank=[33096, 33096, 31655, 31913, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=22485 max_load_reduction_pct=0.404545 moved_to_replicas=46534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=42 tokens_per_ep_rank=[9100, 14647, 16862, 55622, 37406, 52289, 41450, 34768] avg=32768.0000 max=55622 max_rank=3 min=9100 min_rank=0 imbalance_ratio=1.697449 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=42 tokens_per_ep_rank=[33016, 33097, 30551, 33096, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=1 min=30551 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=22525 max_load_reduction_pct=0.404966 moved_to_replicas=56055 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=43 tokens_per_ep_rank=[7294, 51739, 46519, 21691, 41739, 36661, 23145, 33356] avg=32768.0000 max=51739 max_rank=1 min=7294 min_rank=0 imbalance_ratio=1.578949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=43 tokens_per_ep_rank=[30473, 33095, 33096, 33095, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30473 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=18642 max_load_reduction_pct=0.360308 moved_to_replicas=44534 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=44 tokens_per_ep_rank=[27554, 19252, 67564, 25247, 41484, 25931, 34360, 20752] avg=32768.0000 max=67564 max_rank=2 min=19252 min_rank=1 imbalance_ratio=2.061890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=44 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 33096, 31906, 33096, 31663] avg=32768.0000 max=33096 max_rank=0 min=31663 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34468 max_load_reduction_pct=0.510153 moved_to_replicas=44120 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=45 tokens_per_ep_rank=[24155, 41517, 16878, 38293, 29885, 39155, 45949, 26312] avg=32768.0000 max=45949 max_rank=6 min=16878 min_rank=2 imbalance_ratio=1.402252 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=45 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 30472, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12852 max_load_reduction_pct=0.279701 moved_to_replicas=32530 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=46 tokens_per_ep_rank=[33954, 21400, 25260, 22616, 36692, 27264, 46040, 48918] avg=32768.0000 max=48918 max_rank=7 min=21400 min_rank=1 imbalance_ratio=1.492859 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=48 total_expected_recv=1573344 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341376 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=48 total_expected_recv=1573610 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=338432 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=46 tokens_per_ep_rank=[33096, 33096, 33096, 31850, 33096, 31718, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31718 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=15822 max_load_reduction_pct=0.323439 moved_to_replicas=33220 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=47 tokens_per_ep_rank=[26964, 26512, 45709, 40416, 26453, 41859, 24635, 29596] avg=32768.0000 max=45709 max_rank=2 min=24635 min_rank=6 imbalance_ratio=1.394928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=47 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30605, 33096, 33095, 32963] avg=32768.0000 max=33097 max_rank=2 min=30605 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12612 max_load_reduction_pct=0.275919 moved_to_replicas=28696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=48 tokens_per_ep_rank=[41850, 46161, 18571, 21850, 40697, 25801, 39415, 27799] avg=32768.0000 max=46161 max_rank=1 min=18571 min_rank=2 imbalance_ratio=1.408722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=48 tokens_per_ep_rank=[33096, 33096, 31621, 31947, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31621 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13065 max_load_reduction_pct=0.283031 moved_to_replicas=35739 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=48 total_expected_recv=1571774 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=340736 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=48 total_expected_recv=1579862 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=335360 peak_demand=33098 required_capacity=1.010071 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=48 total_expected_recv=1568195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=48 total_expected_recv=1569671 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=48 total_expected_recv=1577775 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=335872 peak_demand=33097 required_capacity=1.010040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=48 total_expected_recv=1568681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33097 required_capacity=1.010040 - [2026-08-26 15:01:19] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 1250.1 | throughput per GPU (TFLOP/s/GPU): 71.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095473E+01 | loss scale: 1.0 | grad norm: 56.502 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32009 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31559 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32009 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=48 total_expected_recv=1577775 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=335872 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=48 total_expected_recv=1579862 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=335360 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=48 total_expected_recv=1573344 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341376 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=48 total_expected_recv=1569671 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=48 total_expected_recv=1573610 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=338432 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:46:59] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 1199.9 | throughput per GPU (TFLOP/s/GPU): 74.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095473E+01 | loss scale: 1.0 | grad norm: 56.507 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32028 expected_overflow=0 exact_recv_per_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31540 expected_overflow=0 exact_recv_per_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32028 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31540 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31559 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=31527 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32040 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=31527 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32051 expected_overflow=0 exact_recv_per_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32051 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32040 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=30803 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=31516 expected_overflow=0 exact_recv_per_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=30803 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=31516 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=30690 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=30690 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=30469 expected_overflow=0 exact_recv_per_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=30469 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=31760 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=31760 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32093 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=31775 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=31775 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32037 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32093 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32037 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=30915 expected_overflow=0 exact_recv_per_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=30915 actual_valid_recv=32128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=31007 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=31007 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=30471 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=30471 actual_valid_recv=30976 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=30976 slack_slots=9984 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=30761 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=30761 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=30961 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32001 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=30961 actual_valid_recv=32000 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32250 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32250 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=31876 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=31876 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32001 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=31781 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=31781 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34816 slack_slots=6144 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=40960 expected_recv=30713 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=40960 expected_recv=30713 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=30562 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=30562 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=40960 expected_recv=31006 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=40960 expected_recv=31006 actual_valid_recv=32384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=40960 expected_recv=30800 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=40960 expected_recv=30770 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=40960 expected_recv=30800 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=40960 expected_recv=30770 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31815 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31815 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=40960 expected_recv=30515 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=40960 expected_recv=30515 actual_valid_recv=31360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31755 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31755 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31541 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=40960 expected_recv=31541 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32027 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32027 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=40960 expected_recv=30635 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=40960 expected_recv=30635 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=40960 expected_recv=30959 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=40960 expected_recv=30959 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=40960 expected_recv=31041 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=40960 expected_recv=30899 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=40960 expected_recv=31041 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=40960 expected_recv=30899 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=40960 expected_recv=30896 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=40960 expected_recv=30896 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=40960 expected_recv=30661 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=40960 expected_recv=30661 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32149 expected_overflow=0 exact_recv_per_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=40960 expected_recv=30638 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32082 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=40960 expected_recv=30638 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32082 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32149 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32080 expected_overflow=0 exact_recv_per_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32080 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32120 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=40960 expected_recv=31974 expected_overflow=0 exact_recv_per_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=40960 expected_recv=31974 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32120 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32199 expected_overflow=0 exact_recv_per_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32043 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32043 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32326 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32326 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31653 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31653 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32199 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31943 expected_overflow=0 exact_recv_per_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31626 expected_overflow=0 exact_recv_per_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31943 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=40960 expected_recv=31629 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=40960 expected_recv=31577 expected_overflow=0 exact_recv_per_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31626 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31915 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=40960 expected_recv=31629 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=40960 expected_recv=31577 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=40960 expected_recv=31915 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32325 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32114 expected_overflow=0 exact_recv_per_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32059 expected_overflow=0 exact_recv_per_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32114 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=40960 expected_recv=31912 expected_overflow=0 exact_recv_per_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32325 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32059 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=40960 expected_recv=31912 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=40960 expected_recv=30906 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=40960 expected_recv=30906 actual_valid_recv=31488 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=23 calls=1 budget=40960 expected_recv=31493 expected_overflow=0 exact_recv_per_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32064 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32064 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=40960 expected_recv=31944 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=23 calls=1 budget=40960 expected_recv=31518 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=40960 expected_recv=31944 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=23 calls=1 budget=40960 expected_recv=31518 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=23 calls=1 budget=40960 expected_recv=31493 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=40960 expected_recv=30884 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=40960 expected_recv=30884 actual_valid_recv=31232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32367 expected_overflow=0 exact_recv_per_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=25 calls=1 budget=40960 expected_recv=31451 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=25 calls=1 budget=40960 expected_recv=31451 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=24 calls=1 budget=40960 expected_recv=31214 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=24 calls=1 budget=40960 expected_recv=31214 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=24 calls=1 budget=40960 expected_recv=31165 expected_overflow=0 exact_recv_per_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=24 calls=1 budget=40960 expected_recv=31165 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=26 calls=1 budget=40960 expected_recv=30745 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=25 calls=1 budget=40960 expected_recv=31250 expected_overflow=0 exact_recv_per_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=25 calls=1 budget=40960 expected_recv=31250 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=26 calls=1 budget=40960 expected_recv=30850 expected_overflow=0 exact_recv_per_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=26 calls=1 budget=40960 expected_recv=30850 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=26 calls=1 budget=40960 expected_recv=30745 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=28 calls=1 budget=40960 expected_recv=30727 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=27 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=28 calls=1 budget=40960 expected_recv=30727 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=28 calls=1 budget=40960 expected_recv=30739 expected_overflow=0 exact_recv_per_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=27 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=28 calls=1 budget=40960 expected_recv=30739 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=27 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=29 calls=1 budget=40960 expected_recv=30765 expected_overflow=0 exact_recv_per_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=29 calls=1 budget=40960 expected_recv=30765 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=29 calls=1 budget=40960 expected_recv=30717 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=29 calls=1 budget=40960 expected_recv=30717 actual_valid_recv=31232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33075 expected_overflow=0 exact_recv_per_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32303 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32303 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=30 calls=1 budget=40960 expected_recv=31942 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=30 calls=1 budget=40960 expected_recv=31942 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=31 calls=1 budget=40960 expected_recv=31996 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32043 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=31 calls=1 budget=40960 expected_recv=31996 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33075 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32043 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=33 calls=1 budget=40960 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=30 calls=1 budget=40960 expected_recv=31914 expected_overflow=0 exact_recv_per_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32307 expected_overflow=0 exact_recv_per_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=30 calls=1 budget=40960 expected_recv=31914 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32307 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32024 expected_overflow=0 exact_recv_per_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32024 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=33 calls=1 budget=40960 expected_recv=30473 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32018 expected_overflow=0 exact_recv_per_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32018 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=32 calls=1 budget=40960 expected_recv=30978 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=32 calls=1 budget=40960 expected_recv=30978 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=34 calls=1 budget=40960 expected_recv=31150 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=33 calls=1 budget=40960 expected_recv=31069 expected_overflow=0 exact_recv_per_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33076 expected_overflow=0 exact_recv_per_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=33 calls=1 budget=40960 expected_recv=31069 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=34 calls=1 budget=40960 expected_recv=31203 expected_overflow=0 exact_recv_per_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33076 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=34 calls=1 budget=40960 expected_recv=31203 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=34 calls=1 budget=40960 expected_recv=31150 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=32 calls=1 budget=40960 expected_recv=30709 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=32 calls=1 budget=40960 expected_recv=30709 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=35 calls=1 budget=40960 expected_recv=31641 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32418 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=35 calls=1 budget=40960 expected_recv=31641 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32418 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33049 expected_overflow=0 exact_recv_per_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33049 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32031 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=35 calls=1 budget=40960 expected_recv=31428 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32365 expected_overflow=0 exact_recv_per_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32365 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=35 calls=1 budget=40960 expected_recv=31428 actual_valid_recv=32128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32031 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31955 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31955 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=36 calls=1 budget=40960 expected_recv=31624 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=36 calls=1 budget=40960 expected_recv=31624 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32004 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32004 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33037 expected_overflow=0 exact_recv_per_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33037 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31613 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31613 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=39 calls=1 budget=40960 expected_recv=31899 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32514 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=39 calls=1 budget=40960 expected_recv=31899 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=36 calls=1 budget=40960 expected_recv=31638 expected_overflow=0 exact_recv_per_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31978 expected_overflow=0 exact_recv_per_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=36 calls=1 budget=40960 expected_recv=31638 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31978 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=36 calls=1 budget=40960 expected_recv=31978 expected_overflow=0 exact_recv_per_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32318 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=36 calls=1 budget=40960 expected_recv=31978 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32318 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33020 expected_overflow=0 exact_recv_per_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33020 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31667 expected_overflow=0 exact_recv_per_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=37 calls=1 budget=40960 expected_recv=31667 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=38 calls=1 budget=40960 expected_recv=31258 expected_overflow=0 exact_recv_per_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=38 calls=1 budget=40960 expected_recv=31258 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=37 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=38 calls=1 budget=40960 expected_recv=31454 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=38 calls=1 budget=40960 expected_recv=31454 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32090 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32964 expected_overflow=0 exact_recv_per_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32090 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32964 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=39 calls=1 budget=40960 expected_recv=30606 expected_overflow=0 exact_recv_per_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=39 calls=1 budget=40960 expected_recv=30606 actual_valid_recv=31232 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=38 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=39 calls=1 budget=40960 expected_recv=31668 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=39 calls=1 budget=40960 expected_recv=31668 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31991 expected_overflow=0 exact_recv_per_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31991 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33008 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31756 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31811 expected_overflow=0 exact_recv_per_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33008 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31756 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31811 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31479 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=40 calls=1 budget=40960 expected_recv=31479 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33015 expected_overflow=0 exact_recv_per_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31929 expected_overflow=0 exact_recv_per_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33015 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31929 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=42 calls=1 budget=40960 expected_recv=31138 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31855 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31976 expected_overflow=0 exact_recv_per_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31746 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31746 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=42 calls=1 budget=40960 expected_recv=31115 expected_overflow=0 exact_recv_per_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=42 calls=1 budget=40960 expected_recv=31115 actual_valid_recv=31616 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=42 calls=1 budget=40960 expected_recv=31138 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31855 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=41 calls=1 budget=40960 expected_recv=31976 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=44 calls=1 budget=40960 expected_recv=31595 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=44 calls=1 budget=40960 expected_recv=31595 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=44 calls=1 budget=40960 expected_recv=31945 expected_overflow=0 exact_recv_per_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33099 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=44 calls=1 budget=40960 expected_recv=31945 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33099 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31780 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31780 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32489 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32489 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32214 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=46 calls=1 budget=40960 expected_recv=31354 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32214 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=46 calls=1 budget=40960 expected_recv=31354 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32960 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32960 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=45 calls=1 budget=40960 expected_recv=31020 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=45 calls=1 budget=40960 expected_recv=31020 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=45 calls=1 budget=40960 expected_recv=30962 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=45 calls=1 budget=40960 expected_recv=30962 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33077 expected_overflow=0 exact_recv_per_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33077 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=44 calls=1 budget=40960 expected_recv=31642 expected_overflow=0 exact_recv_per_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=44 calls=1 budget=40960 expected_recv=31642 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31822 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=43 calls=1 budget=40960 expected_recv=31822 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=46 calls=1 budget=40960 expected_recv=31365 expected_overflow=0 exact_recv_per_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32392 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=46 calls=1 budget=40960 expected_recv=31365 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32392 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=47 calls=1 budget=40960 expected_recv=31329 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=47 calls=1 budget=40960 expected_recv=31329 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33080 expected_overflow=0 exact_recv_per_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33080 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[40756, 32009, 29546, 22479, 41497, 26692, 37770, 31395] avg=32768.0000 max=41497 max_rank=4 min=22479 min_rank=3 imbalance_ratio=1.266388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32009, 31559, 33095, 33096, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31559 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8400 max_load_reduction_pct=0.202424 moved_to_replicas=20735 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[25348, 32040, 50112, 13132, 6014, 67596, 28348, 39554] avg=32768.0000 max=67596 max_rank=5 min=6014 min_rank=4 imbalance_ratio=2.062866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32040, 33096, 31527, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31527 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34499 max_load_reduction_pct=0.510370 moved_to_replicas=57974 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[64667, 63575, 27261, 16498, 18540, 11944, 21059, 38600] avg=32768.0000 max=64667 max_rank=0 min=11944 min_rank=5 imbalance_ratio=1.973480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32764, 30803, 33095, 33096, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30803 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31570 max_load_reduction_pct=0.488193 moved_to_replicas=67554 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[6962, 6711, 65457, 56469, 8918, 14752, 41635, 61240] avg=32768.0000 max=65457 max_rank=2 min=6711 min_rank=1 imbalance_ratio=1.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[30472, 33096, 33096, 33096, 33095, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32360 max_load_reduction_pct=0.494370 moved_to_replicas=92417 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[42669, 16005, 1835, 78359, 32813, 1521, 34129, 54813] avg=32768.0000 max=78359 max_rank=3 min=1521 min_rank=5 imbalance_ratio=2.391327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[33095, 33097, 32093, 33095, 32813, 31760, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31760 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45262 max_load_reduction_pct=0.577624 moved_to_replicas=77586 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[63632, 8289, 296, 8045, 83255, 1465, 50329, 46833] avg=32768.0000 max=83255 max_rank=4 min=296 min_rank=2 imbalance_ratio=2.540741 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[33096, 31007, 32561, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31007 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50159 max_load_reduction_pct=0.602474 moved_to_replicas=111665 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[85760, 17647, 35472, 23029, 22964, 6850, 7363, 63059] avg=32768.0000 max=85760 max_rank=0 min=6850 min_rank=5 imbalance_ratio=2.617188 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 30471, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30471 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=52663 max_load_reduction_pct=0.614074 moved_to_replicas=85003 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[71195, 21968, 45052, 2280, 35757, 38240, 8861, 38791] avg=32768.0000 max=71195 max_rank=0 min=2280 min_rank=3 imbalance_ratio=2.172699 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32807, 33096, 33096, 33096, 33096, 30761, 33096] avg=32768.0000 max=33096 max_rank=0 min=30761 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38099 max_load_reduction_pct=0.535136 moved_to_replicas=63555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[7964, 50257, 31781, 45180, 48784, 12202, 3303, 62673] avg=32768.0000 max=62673 max_rank=7 min=3303 min_rank=6 imbalance_ratio=1.912628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31781, 33096, 33098, 32001, 32880, 33096] avg=32768.0000 max=33098 max_rank=4 min=31781 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29575 max_load_reduction_pct=0.471894 moved_to_replicas=74510 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[16549, 18131, 63472, 23499, 40672, 58527, 10899, 30395] avg=32768.0000 max=63472 max_rank=2 min=10899 min_rank=6 imbalance_ratio=1.937012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 33095, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30375 max_load_reduction_pct=0.478557 moved_to_replicas=63383 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[64961, 12561, 33623, 32854, 2017, 34801, 62199, 19128] avg=32768.0000 max=64961 max_rank=0 min=2017 min_rank=4 imbalance_ratio=1.982452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33097, 33096, 32854, 33096, 33096, 33096, 30713] avg=32768.0000 max=33097 max_rank=1 min=30713 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31864 max_load_reduction_pct=0.490510 moved_to_replicas=63200 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[35266, 13599, 51303, 56377, 15400, 32799, 24172, 33228] avg=32768.0000 max=56377 max_rank=3 min=13599 min_rank=1 imbalance_ratio=1.720490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33095, 33097, 33096, 32799, 30770, 33096] avg=32768.0000 max=33097 max_rank=3 min=30770 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23280 max_load_reduction_pct=0.412934 moved_to_replicas=43790 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[33882, 5027, 331, 27757, 84427, 30442, 241, 80037] avg=32768.0000 max=84427 max_rank=4 min=241 min_rank=6 imbalance_ratio=2.576508 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33094, 31755, 33097, 33096, 31815, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31755 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=51330 max_load_reduction_pct=0.607981 moved_to_replicas=99058 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[7288, 3850, 51929, 3048, 60697, 44562, 75543, 15227] avg=32768.0000 max=75543 max_rank=6 min=3048 min_rank=3 imbalance_ratio=2.305389 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33095, 33097, 33053, 33096, 33096, 33096, 30515] avg=32768.0000 max=33097 max_rank=2 min=30515 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42446 max_load_reduction_pct=0.561879 moved_to_replicas=100347 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[35057, 34685, 24300, 12146, 24119, 55450, 43860, 32527] avg=32768.0000 max=55450 max_rank=5 min=12146 min_rank=3 imbalance_ratio=1.692200 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33094, 31041, 33098, 33096, 32527] avg=32768.0000 max=33098 max_rank=5 min=31041 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=22352 max_load_reduction_pct=0.403102 moved_to_replicas=36668 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[74533, 23428, 66668, 26816, 3922, 27631, 37, 39109] avg=32768.0000 max=74533 max_rank=0 min=37 min_rank=6 imbalance_ratio=2.274567 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33097, 33096, 30899, 33094, 33098, 32669, 33094] avg=32768.0000 max=33098 max_rank=5 min=30899 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41435 max_load_reduction_pct=0.555928 moved_to_replicas=81022 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[35625, 2932, 70815, 49648, 350, 42719, 15855, 44200] avg=32768.0000 max=70815 max_rank=2 min=350 min_rank=4 imbalance_ratio=2.161102 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33097, 33096, 32931, 33096, 30638, 33095] avg=32768.0000 max=33097 max_rank=2 min=30638 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37718 max_load_reduction_pct=0.532627 moved_to_replicas=77527 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[18315, 34029, 31110, 42568, 61312, 5878, 36394, 32538] avg=32768.0000 max=61312 max_rank=4 min=5878 min_rank=5 imbalance_ratio=1.871094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[32082, 33096, 32043, 33097, 33095, 33097, 33096, 32538] avg=32768.0000 max=33097 max_rank=3 min=32043 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28215 max_load_reduction_pct=0.460187 moved_to_replicas=41919 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[3455, 61761, 4047, 32326, 31685, 33857, 35329, 59684] avg=32768.0000 max=61761 max_rank=1 min=3455 min_rank=0 imbalance_ratio=1.884796 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[32120, 33096, 32868, 32326, 32446, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32120 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28665 max_load_reduction_pct=0.464128 moved_to_replicas=58247 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[39771, 650, 117953, 40615, 8800, 27923, 6943, 19489] avg=32768.0000 max=117953 max_rank=2 min=650 min_rank=1 imbalance_ratio=3.599640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31915, 33095, 33096, 33096, 33095, 33097, 31653] avg=32768.0000 max=33097 max_rank=0 min=31653 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84856 max_load_reduction_pct=0.719405 moved_to_replicas=99051 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[31629, 60388, 36110, 16234, 192, 55183, 4762, 57646] avg=32768.0000 max=60388 max_rank=1 min=192 min_rank=4 imbalance_ratio=1.842896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[31629, 33095, 33096, 33096, 32710, 33096, 32325, 33097] avg=32768.0000 max=33097 max_rank=7 min=31629 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27291 max_load_reduction_pct=0.451928 moved_to_replicas=76943 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[2812, 66691, 13297, 75131, 24146, 39842, 39926, 299] avg=32768.0000 max=75131 max_rank=3 min=299 min_rank=7 imbalance_ratio=2.292816 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[33096, 33096, 32064, 33096, 31944, 33095, 33096, 32657] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=42035 max_load_reduction_pct=0.559489 moved_to_replicas=89206 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[53989, 21111, 43429, 45108, 30884, 54379, 844, 12400] avg=32768.0000 max=54379 max_rank=5 min=844 min_rank=6 imbalance_ratio=1.659515 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 30884, 33095, 32685, 33096] avg=32768.0000 max=33098 max_rank=1 min=30884 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21281 max_load_reduction_pct=0.391346 moved_to_replicas=64521 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[31013, 32779, 30405, 27721, 312, 75468, 23521, 40925] avg=32768.0000 max=75468 max_rank=5 min=312 min_rank=4 imbalance_ratio=2.303101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[31518, 32779, 33096, 33096, 32367, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31518 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42371 max_load_reduction_pct=0.561443 moved_to_replicas=50201 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=25 tokens_per_ep_rank=[6151, 31214, 15585, 12220, 206, 117595, 32803, 46370] avg=32768.0000 max=117595 max_rank=5 min=206 min_rank=4 imbalance_ratio=3.588715 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31214, 33095, 32884, 32860, 33097, 32803, 33095] avg=32768.0000 max=33097 max_rank=5 min=31214 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=84498 max_load_reduction_pct=0.718551 moved_to_replicas=97773 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=26 tokens_per_ep_rank=[51240, 7167, 32810, 52093, 68281, 18641, 461, 31451] avg=32768.0000 max=68281 max_rank=4 min=461 min_rank=6 imbalance_ratio=2.083771 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32435, 32810, 33096, 33096, 33096, 33063, 31451] avg=32768.0000 max=33097 max_rank=0 min=31451 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=72326 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=27 tokens_per_ep_rank=[28951, 29935, 13471, 21832, 38557, 40766, 70864, 17768] avg=32768.0000 max=70864 max_rank=6 min=13471 min_rank=2 imbalance_ratio=2.162598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=27 tokens_per_ep_rank=[30745, 33096, 33096, 33095, 33096, 33097, 33096, 32823] avg=32768.0000 max=33097 max_rank=5 min=30745 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37767 max_load_reduction_pct=0.532950 moved_to_replicas=50899 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=28 tokens_per_ep_rank=[6454, 4635, 14687, 81982, 15875, 47903, 28206, 62402] avg=32768.0000 max=81982 max_rank=3 min=4635 min_rank=1 imbalance_ratio=2.501892 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30472, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=48886 max_load_reduction_pct=0.596302 moved_to_replicas=92999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=29 tokens_per_ep_rank=[71, 32841, 22246, 65065, 68178, 26928, 23510, 23305] avg=32768.0000 max=68178 max_rank=4 min=71 min_rank=0 imbalance_ratio=2.080627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=29 tokens_per_ep_rank=[30727, 32841, 33097, 33096, 33095, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30727 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35081 max_load_reduction_pct=0.514550 moved_to_replicas=67051 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=30 tokens_per_ep_rank=[987, 57338, 323, 94998, 44098, 3756, 6595, 54049] avg=32768.0000 max=94998 max_rank=3 min=323 min_rank=2 imbalance_ratio=2.899109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=30 tokens_per_ep_rank=[30717, 33096, 32851, 33096, 33097, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=30717 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=61901 max_load_reduction_pct=0.651603 moved_to_replicas=118099 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=31 tokens_per_ep_rank=[42126, 28886, 81135, 1730, 74570, 220, 32419, 1058] avg=32768.0000 max=81135 max_rank=2 min=220 min_rank=5 imbalance_ratio=2.476044 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=31 tokens_per_ep_rank=[33094, 31942, 33096, 33096, 33097, 32303, 32419, 33097] avg=32768.0000 max=33097 max_rank=4 min=31942 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48038 max_load_reduction_pct=0.592075 moved_to_replicas=98543 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=32 tokens_per_ep_rank=[785, 113, 87900, 40280, 66907, 56169, 854, 9136] avg=32768.0000 max=87900 max_rank=2 min=113 min_rank=1 imbalance_ratio=2.682495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=32 tokens_per_ep_rank=[32626, 32043, 33095, 33096, 33096, 33098, 31996, 33094] avg=32768.0000 max=33098 max_rank=5 min=31996 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=54802 max_load_reduction_pct=0.623458 moved_to_replicas=118872 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=33 tokens_per_ep_rank=[21406, 57429, 30138, 32590, 54403, 41800, 24213, 165] avg=32768.0000 max=57429 max_rank=1 min=165 min_rank=7 imbalance_ratio=1.752594 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33096, 30978, 32590, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=30978 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24332 max_load_reduction_pct=0.423688 moved_to_replicas=54344 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=34 tokens_per_ep_rank=[25708, 33141, 61630, 24590, 36182, 18456, 37430, 25007] avg=32768.0000 max=61630 max_rank=2 min=18456 min_rank=5 imbalance_ratio=1.880798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=34 tokens_per_ep_rank=[33094, 33096, 33096, 33095, 33096, 33096, 33098, 30473] avg=32768.0000 max=33098 max_rank=6 min=30473 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28532 max_load_reduction_pct=0.462956 moved_to_replicas=35999 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=35 tokens_per_ep_rank=[29106, 9303, 32418, 41235, 2739, 15039, 11417, 120887] avg=32768.0000 max=120887 max_rank=7 min=2739 min_rank=4 imbalance_ratio=3.689178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=35 tokens_per_ep_rank=[31150, 33097, 32418, 33095, 33096, 33098, 33095, 33095] avg=32768.0000 max=33098 max_rank=5 min=31150 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=87789 max_load_reduction_pct=0.726207 moved_to_replicas=95930 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=36 tokens_per_ep_rank=[44396, 49867, 5539, 25085, 32993, 59587, 7894, 36783] avg=32768.0000 max=59587 max_rank=5 min=5539 min_rank=2 imbalance_ratio=1.818451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33097, 32031, 31641, 32993, 33095, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31641 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26490 max_load_reduction_pct=0.444560 moved_to_replicas=58249 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=37 tokens_per_ep_rank=[5467, 27327, 33440, 50869, 33037, 8637, 31280, 72087] avg=32768.0000 max=72087 max_rank=7 min=5467 min_rank=0 imbalance_ratio=2.199921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=37 tokens_per_ep_rank=[33097, 32004, 33096, 33094, 33037, 33096, 31624, 33096] avg=32768.0000 max=33097 max_rank=0 min=31624 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38990 max_load_reduction_pct=0.540874 moved_to_replicas=57108 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=38 tokens_per_ep_rank=[214, 48111, 13042, 66331, 1001, 30672, 18832, 83941] avg=32768.0000 max=83941 max_rank=7 min=214 min_rank=0 imbalance_ratio=2.561676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=38 tokens_per_ep_rank=[31955, 33096, 33096, 33096, 33095, 33096, 31613, 33097] avg=32768.0000 max=33097 max_rank=7 min=31613 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50844 max_load_reduction_pct=0.605711 moved_to_replicas=99095 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=39 tokens_per_ep_rank=[1329, 31454, 37622, 26610, 25935, 64462, 52241, 22491] avg=32768.0000 max=64462 max_rank=5 min=1329 min_rank=0 imbalance_ratio=1.967224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=39 tokens_per_ep_rank=[32695, 31454, 33097, 32514, 33097, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31454 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31365 max_load_reduction_pct=0.486566 moved_to_replicas=55037 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=40 tokens_per_ep_rank=[57217, 580, 20110, 48636, 17260, 64184, 49750, 4407] avg=32768.0000 max=64184 max_rank=5 min=580 min_rank=1 imbalance_ratio=1.958740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=40 tokens_per_ep_rank=[33097, 31668, 33096, 33096, 33096, 33096, 33096, 31899] avg=32768.0000 max=33097 max_rank=0 min=31668 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=31087 max_load_reduction_pct=0.484342 moved_to_replicas=87403 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=41 tokens_per_ep_rank=[20199, 13602, 58225, 7554, 31281, 40779, 7522, 82982] avg=32768.0000 max=82982 max_rank=7 min=7522 min_rank=6 imbalance_ratio=2.532410 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=41 tokens_per_ep_rank=[33096, 31756, 33096, 33098, 31811, 33096, 33094, 33097] avg=32768.0000 max=33098 max_rank=3 min=31756 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=49884 max_load_reduction_pct=0.601142 moved_to_replicas=82698 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=42 tokens_per_ep_rank=[17307, 1578, 6545, 116729, 423, 67614, 40854, 11094] avg=32768.0000 max=116729 max_rank=3 min=423 min_rank=4 imbalance_ratio=3.562286 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=42 tokens_per_ep_rank=[32834, 31855, 33095, 33097, 31976, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=3 min=31855 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83632 max_load_reduction_pct=0.716463 moved_to_replicas=125909 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=43 tokens_per_ep_rank=[33008, 41323, 7322, 200, 64884, 586, 74519, 40302] avg=32768.0000 max=74519 max_rank=6 min=200 min_rank=3 imbalance_ratio=2.274139 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=43 tokens_per_ep_rank=[33008, 33095, 33097, 32519, 33096, 31138, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31138 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41422 max_load_reduction_pct=0.555858 moved_to_replicas=88644 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=44 tokens_per_ep_rank=[33501, 7500, 31780, 24131, 25898, 58708, 7568, 73058] avg=32768.0000 max=73058 max_rank=7 min=7500 min_rank=1 imbalance_ratio=2.229553 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33095, 31780, 33095, 31787, 33097, 33099, 33095] avg=32768.0000 max=33099 max_rank=6 min=31780 min_rank=2 imbalance_ratio=1.010101 max_load_reduction=39959 max_load_reduction_pct=0.546949 moved_to_replicas=65979 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=45 tokens_per_ep_rank=[16497, 58245, 65933, 66311, 9100, 18657, 21018, 6383] avg=32768.0000 max=66311 max_rank=3 min=6383 min_rank=7 imbalance_ratio=2.023651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=45 tokens_per_ep_rank=[31595, 33096, 33096, 33097, 33094, 33096, 32489, 32581] avg=32768.0000 max=33097 max_rank=3 min=31595 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33214 max_load_reduction_pct=0.500882 moved_to_replicas=91201 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=46 tokens_per_ep_rank=[14681, 16498, 33455, 33072, 29172, 33418, 32572, 69276] avg=32768.0000 max=69276 max_rank=7 min=14681 min_rank=0 imbalance_ratio=2.114136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=46 tokens_per_ep_rank=[33097, 33095, 33096, 33072, 31020, 33096, 32572, 33096] avg=32768.0000 max=33097 max_rank=0 min=31020 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36179 max_load_reduction_pct=0.522244 moved_to_replicas=36861 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=47 tokens_per_ep_rank=[147, 28269, 29653, 21098, 26947, 64303, 61845, 29882] avg=32768.0000 max=64303 max_rank=5 min=147 min_rank=0 imbalance_ratio=1.962372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=47 tokens_per_ep_rank=[31354, 33097, 33096, 33096, 33095, 33096, 33096, 32214] avg=32768.0000 max=33097 max_rank=1 min=31354 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31206 max_load_reduction_pct=0.485296 moved_to_replicas=59956 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=48 tokens_per_ep_rank=[16454, 33080, 7909, 19163, 50859, 46325, 49603, 38751] avg=32768.0000 max=50859 max_rank=4 min=7909 min_rank=2 imbalance_ratio=1.552094 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=48 tokens_per_ep_rank=[32960, 33080, 31329, 32392, 33095, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31329 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17762 max_load_reduction_pct=0.349240 moved_to_replicas=53154 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=48 total_expected_recv=1570870 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=354688 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=48 total_expected_recv=1574804 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=353024 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=48 total_expected_recv=1566166 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=360704 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=48 total_expected_recv=1581859 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344192 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=48 total_expected_recv=1574781 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=353792 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=48 total_expected_recv=1570851 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=355328 peak_demand=33099 required_capacity=1.010101 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=48 total_expected_recv=1572757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=356608 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=48 total_expected_recv=1570824 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=356224 peak_demand=33098 required_capacity=1.010071 - [2026-08-26 15:01:21] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 1287.9 | throughput per GPU (TFLOP/s/GPU): 69.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008786E+01 | loss scale: 1.0 | grad norm: 180.776 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32557 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=31011 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32557 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32203 expected_overflow=0 exact_recv_per_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32203 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[40797, 32028, 29487, 22498, 41471, 26686, 37792, 31385] avg=32768.0000 max=41471 max_rank=4 min=22498 min_rank=3 imbalance_ratio=1.265594 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31540 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8374 max_load_reduction_pct=0.201924 moved_to_replicas=20772 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[40797, 32028, 29487, 22498, 41471, 26686, 37792, 31385] avg=32768.0000 max=41471 max_rank=4 min=22498 min_rank=3 imbalance_ratio=1.265594 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31540 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8374 max_load_reduction_pct=0.201924 moved_to_replicas=20772 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[25147, 32051, 50054, 13086, 6025, 67514, 28464, 39803] avg=32768.0000 max=67514 max_rank=5 min=6025 min_rank=4 imbalance_ratio=2.060364 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32371 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32371 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[40797, 32028, 29487, 22498, 41471, 26686, 37792, 31385] avg=32768.0000 max=41471 max_rank=4 min=22498 min_rank=3 imbalance_ratio=1.265594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[40797, 32028, 29487, 22498, 41471, 26686, 37792, 31385] avg=32768.0000 max=41471 max_rank=4 min=22498 min_rank=3 imbalance_ratio=1.265594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[40797, 32028, 29487, 22498, 41471, 26686, 37792, 31385] avg=32768.0000 max=41471 max_rank=4 min=22498 min_rank=3 imbalance_ratio=1.265594 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31516 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34417 max_load_reduction_pct=0.509776 moved_to_replicas=58083 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[40797, 32028, 29487, 22498, 41471, 26686, 37792, 31385] avg=32768.0000 max=41471 max_rank=4 min=22498 min_rank=3 imbalance_ratio=1.265594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[25147, 32051, 50054, 13086, 6025, 67514, 28464, 39803] avg=32768.0000 max=67514 max_rank=5 min=6025 min_rank=4 imbalance_ratio=2.060364 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=47 calls=1 budget=40960 expected_recv=31567 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31540 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8374 max_load_reduction_pct=0.201924 moved_to_replicas=20772 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31540 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8374 max_load_reduction_pct=0.201924 moved_to_replicas=20772 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31540 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8374 max_load_reduction_pct=0.201924 moved_to_replicas=20772 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=47 calls=1 budget=40960 expected_recv=31567 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31540 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8374 max_load_reduction_pct=0.201924 moved_to_replicas=20772 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31516 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34417 max_load_reduction_pct=0.509776 moved_to_replicas=58083 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[64928, 63591, 27335, 16491, 18561, 11705, 20895, 38638] avg=32768.0000 max=64928 max_rank=0 min=11705 min_rank=5 imbalance_ratio=1.981445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[40797, 32028, 29487, 22498, 41471, 26686, 37792, 31385] avg=32768.0000 max=41471 max_rank=4 min=22498 min_rank=3 imbalance_ratio=1.265594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[25147, 32051, 50054, 13086, 6025, 67514, 28464, 39803] avg=32768.0000 max=67514 max_rank=5 min=6025 min_rank=4 imbalance_ratio=2.060364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[25147, 32051, 50054, 13086, 6025, 67514, 28464, 39803] avg=32768.0000 max=67514 max_rank=5 min=6025 min_rank=4 imbalance_ratio=2.060364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[25147, 32051, 50054, 13086, 6025, 67514, 28464, 39803] avg=32768.0000 max=67514 max_rank=5 min=6025 min_rank=4 imbalance_ratio=2.060364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30690 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31831 max_load_reduction_pct=0.490251 moved_to_replicas=67869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[25147, 32051, 50054, 13086, 6025, 67514, 28464, 39803] avg=32768.0000 max=67514 max_rank=5 min=6025 min_rank=4 imbalance_ratio=2.060364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31540 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8374 max_load_reduction_pct=0.201924 moved_to_replicas=20772 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[64928, 63591, 27335, 16491, 18561, 11705, 20895, 38638] avg=32768.0000 max=64928 max_rank=0 min=11705 min_rank=5 imbalance_ratio=1.981445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[40797, 32028, 29487, 22498, 41471, 26686, 37792, 31385] avg=32768.0000 max=41471 max_rank=4 min=22498 min_rank=3 imbalance_ratio=1.265594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31516 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34417 max_load_reduction_pct=0.509776 moved_to_replicas=58083 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31516 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34417 max_load_reduction_pct=0.509776 moved_to_replicas=58083 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31516 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34417 max_load_reduction_pct=0.509776 moved_to_replicas=58083 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31516 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34417 max_load_reduction_pct=0.509776 moved_to_replicas=58083 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30690 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31831 max_load_reduction_pct=0.490251 moved_to_replicas=67869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[6857, 6668, 65759, 56214, 9016, 14735, 41611, 61284] avg=32768.0000 max=65759 max_rank=2 min=6668 min_rank=1 imbalance_ratio=2.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[25147, 32051, 50054, 13086, 6025, 67514, 28464, 39803] avg=32768.0000 max=67514 max_rank=5 min=6025 min_rank=4 imbalance_ratio=2.060364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[33096, 32028, 31540, 33096, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31540 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=8374 max_load_reduction_pct=0.201924 moved_to_replicas=20772 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33098 max_rank=1 min=30469 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32661 max_load_reduction_pct=0.496677 moved_to_replicas=92484 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[64928, 63591, 27335, 16491, 18561, 11705, 20895, 38638] avg=32768.0000 max=64928 max_rank=0 min=11705 min_rank=5 imbalance_ratio=1.981445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[64928, 63591, 27335, 16491, 18561, 11705, 20895, 38638] avg=32768.0000 max=64928 max_rank=0 min=11705 min_rank=5 imbalance_ratio=1.981445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[64928, 63591, 27335, 16491, 18561, 11705, 20895, 38638] avg=32768.0000 max=64928 max_rank=0 min=11705 min_rank=5 imbalance_ratio=1.981445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[64928, 63591, 27335, 16491, 18561, 11705, 20895, 38638] avg=32768.0000 max=64928 max_rank=0 min=11705 min_rank=5 imbalance_ratio=1.981445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[6857, 6668, 65759, 56214, 9016, 14735, 41611, 61284] avg=32768.0000 max=65759 max_rank=2 min=6668 min_rank=1 imbalance_ratio=2.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31516 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34417 max_load_reduction_pct=0.509776 moved_to_replicas=58083 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30690 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31831 max_load_reduction_pct=0.490251 moved_to_replicas=67869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30690 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31831 max_load_reduction_pct=0.490251 moved_to_replicas=67869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30690 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31831 max_load_reduction_pct=0.490251 moved_to_replicas=67869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30690 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31831 max_load_reduction_pct=0.490251 moved_to_replicas=67869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[25147, 32051, 50054, 13086, 6025, 67514, 28464, 39803] avg=32768.0000 max=67514 max_rank=5 min=6025 min_rank=4 imbalance_ratio=2.060364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[42644, 15782, 1815, 78434, 32852, 1522, 34152, 54943] avg=32768.0000 max=78434 max_rank=3 min=1522 min_rank=5 imbalance_ratio=2.393616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33098 max_rank=1 min=30469 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32661 max_load_reduction_pct=0.496677 moved_to_replicas=92484 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[64928, 63591, 27335, 16491, 18561, 11705, 20895, 38638] avg=32768.0000 max=64928 max_rank=0 min=11705 min_rank=5 imbalance_ratio=1.981445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45337 max_load_reduction_pct=0.578027 moved_to_replicas=77789 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[33095, 32051, 33097, 31516, 33096, 33097, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31516 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34417 max_load_reduction_pct=0.509776 moved_to_replicas=58083 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[6857, 6668, 65759, 56214, 9016, 14735, 41611, 61284] avg=32768.0000 max=65759 max_rank=2 min=6668 min_rank=1 imbalance_ratio=2.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[6857, 6668, 65759, 56214, 9016, 14735, 41611, 61284] avg=32768.0000 max=65759 max_rank=2 min=6668 min_rank=1 imbalance_ratio=2.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[6857, 6668, 65759, 56214, 9016, 14735, 41611, 61284] avg=32768.0000 max=65759 max_rank=2 min=6668 min_rank=1 imbalance_ratio=2.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[6857, 6668, 65759, 56214, 9016, 14735, 41611, 61284] avg=32768.0000 max=65759 max_rank=2 min=6668 min_rank=1 imbalance_ratio=2.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[42644, 15782, 1815, 78434, 32852, 1522, 34152, 54943] avg=32768.0000 max=78434 max_rank=3 min=1522 min_rank=5 imbalance_ratio=2.393616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30690 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31831 max_load_reduction_pct=0.490251 moved_to_replicas=67869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33098 max_rank=1 min=30469 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32661 max_load_reduction_pct=0.496677 moved_to_replicas=92484 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33098 max_rank=1 min=30469 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32661 max_load_reduction_pct=0.496677 moved_to_replicas=92484 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33098 max_rank=1 min=30469 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32661 max_load_reduction_pct=0.496677 moved_to_replicas=92484 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33098 max_rank=1 min=30469 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32661 max_load_reduction_pct=0.496677 moved_to_replicas=92484 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45337 max_load_reduction_pct=0.578027 moved_to_replicas=77789 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[63364, 8273, 293, 8189, 83368, 1431, 50376, 46850] avg=32768.0000 max=83368 max_rank=4 min=293 min_rank=2 imbalance_ratio=2.544189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[64928, 63591, 27335, 16491, 18561, 11705, 20895, 38638] avg=32768.0000 max=64928 max_rank=0 min=11705 min_rank=5 imbalance_ratio=1.981445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[6857, 6668, 65759, 56214, 9016, 14735, 41611, 61284] avg=32768.0000 max=65759 max_rank=2 min=6668 min_rank=1 imbalance_ratio=2.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30915 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50271 max_load_reduction_pct=0.603001 moved_to_replicas=111574 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[42644, 15782, 1815, 78434, 32852, 1522, 34152, 54943] avg=32768.0000 max=78434 max_rank=3 min=1522 min_rank=5 imbalance_ratio=2.393616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[33096, 33096, 32877, 30690, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30690 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=31831 max_load_reduction_pct=0.490251 moved_to_replicas=67869 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[42644, 15782, 1815, 78434, 32852, 1522, 34152, 54943] avg=32768.0000 max=78434 max_rank=3 min=1522 min_rank=5 imbalance_ratio=2.393616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[42644, 15782, 1815, 78434, 32852, 1522, 34152, 54943] avg=32768.0000 max=78434 max_rank=3 min=1522 min_rank=5 imbalance_ratio=2.393616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[42644, 15782, 1815, 78434, 32852, 1522, 34152, 54943] avg=32768.0000 max=78434 max_rank=3 min=1522 min_rank=5 imbalance_ratio=2.393616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[63364, 8273, 293, 8189, 83368, 1431, 50376, 46850] avg=32768.0000 max=83368 max_rank=4 min=293 min_rank=2 imbalance_ratio=2.544189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33098 max_rank=1 min=30469 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32661 max_load_reduction_pct=0.496677 moved_to_replicas=92484 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45337 max_load_reduction_pct=0.578027 moved_to_replicas=77789 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45337 max_load_reduction_pct=0.578027 moved_to_replicas=77789 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45337 max_load_reduction_pct=0.578027 moved_to_replicas=77789 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45337 max_load_reduction_pct=0.578027 moved_to_replicas=77789 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[85865, 17501, 35545, 22706, 23216, 6992, 7190, 63129] avg=32768.0000 max=85865 max_rank=0 min=6992 min_rank=5 imbalance_ratio=2.620392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30915 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50271 max_load_reduction_pct=0.603001 moved_to_replicas=111574 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[6857, 6668, 65759, 56214, 9016, 14735, 41611, 61284] avg=32768.0000 max=65759 max_rank=2 min=6668 min_rank=1 imbalance_ratio=2.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[42644, 15782, 1815, 78434, 32852, 1522, 34152, 54943] avg=32768.0000 max=78434 max_rank=3 min=1522 min_rank=5 imbalance_ratio=2.393616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[63364, 8273, 293, 8189, 83368, 1431, 50376, 46850] avg=32768.0000 max=83368 max_rank=4 min=293 min_rank=2 imbalance_ratio=2.544189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52769 max_load_reduction_pct=0.614558 moved_to_replicas=85251 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[30469, 33098, 33096, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33098 max_rank=1 min=30469 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32661 max_load_reduction_pct=0.496677 moved_to_replicas=92484 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[63364, 8273, 293, 8189, 83368, 1431, 50376, 46850] avg=32768.0000 max=83368 max_rank=4 min=293 min_rank=2 imbalance_ratio=2.544189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[63364, 8273, 293, 8189, 83368, 1431, 50376, 46850] avg=32768.0000 max=83368 max_rank=4 min=293 min_rank=2 imbalance_ratio=2.544189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[63364, 8273, 293, 8189, 83368, 1431, 50376, 46850] avg=32768.0000 max=83368 max_rank=4 min=293 min_rank=2 imbalance_ratio=2.544189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[85865, 17501, 35545, 22706, 23216, 6992, 7190, 63129] avg=32768.0000 max=85865 max_rank=0 min=6992 min_rank=5 imbalance_ratio=2.620392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45337 max_load_reduction_pct=0.578027 moved_to_replicas=77789 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30915 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50271 max_load_reduction_pct=0.603001 moved_to_replicas=111574 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[71228, 22082, 45320, 2291, 35931, 37980, 8575, 38737] avg=32768.0000 max=71228 max_rank=0 min=2291 min_rank=3 imbalance_ratio=2.173706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30915 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50271 max_load_reduction_pct=0.603001 moved_to_replicas=111574 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30915 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50271 max_load_reduction_pct=0.603001 moved_to_replicas=111574 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30915 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50271 max_load_reduction_pct=0.603001 moved_to_replicas=111574 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52769 max_load_reduction_pct=0.614558 moved_to_replicas=85251 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[42644, 15782, 1815, 78434, 32852, 1522, 34152, 54943] avg=32768.0000 max=78434 max_rank=3 min=1522 min_rank=5 imbalance_ratio=2.393616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[85865, 17501, 35545, 22706, 23216, 6992, 7190, 63129] avg=32768.0000 max=85865 max_rank=0 min=6992 min_rank=5 imbalance_ratio=2.620392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] avg=32768.0000 max=33096 max_rank=0 min=30961 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38132 max_load_reduction_pct=0.535351 moved_to_replicas=63716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[63364, 8273, 293, 8189, 83368, 1431, 50376, 46850] avg=32768.0000 max=83368 max_rank=4 min=293 min_rank=2 imbalance_ratio=2.544189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[33097, 33096, 32037, 33095, 32852, 31775, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=45337 max_load_reduction_pct=0.578027 moved_to_replicas=77789 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[85865, 17501, 35545, 22706, 23216, 6992, 7190, 63129] avg=32768.0000 max=85865 max_rank=0 min=6992 min_rank=5 imbalance_ratio=2.620392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[85865, 17501, 35545, 22706, 23216, 6992, 7190, 63129] avg=32768.0000 max=85865 max_rank=0 min=6992 min_rank=5 imbalance_ratio=2.620392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[85865, 17501, 35545, 22706, 23216, 6992, 7190, 63129] avg=32768.0000 max=85865 max_rank=0 min=6992 min_rank=5 imbalance_ratio=2.620392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[71228, 22082, 45320, 2291, 35931, 37980, 8575, 38737] avg=32768.0000 max=71228 max_rank=0 min=2291 min_rank=3 imbalance_ratio=2.173706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52769 max_load_reduction_pct=0.614558 moved_to_replicas=85251 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30915 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50271 max_load_reduction_pct=0.603001 moved_to_replicas=111574 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[7953, 50341, 31876, 45242, 48802, 12294, 3327, 62309] avg=32768.0000 max=62309 max_rank=7 min=3327 min_rank=6 imbalance_ratio=1.901520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52769 max_load_reduction_pct=0.614558 moved_to_replicas=85251 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52769 max_load_reduction_pct=0.614558 moved_to_replicas=85251 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52769 max_load_reduction_pct=0.614558 moved_to_replicas=85251 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] avg=32768.0000 max=33096 max_rank=0 min=30961 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38132 max_load_reduction_pct=0.535351 moved_to_replicas=63716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[63364, 8273, 293, 8189, 83368, 1431, 50376, 46850] avg=32768.0000 max=83368 max_rank=4 min=293 min_rank=2 imbalance_ratio=2.544189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] avg=32768.0000 max=33096 max_rank=0 min=31876 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29213 max_load_reduction_pct=0.468841 moved_to_replicas=74310 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[71228, 22082, 45320, 2291, 35931, 37980, 8575, 38737] avg=32768.0000 max=71228 max_rank=0 min=2291 min_rank=3 imbalance_ratio=2.173706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[85865, 17501, 35545, 22706, 23216, 6992, 7190, 63129] avg=32768.0000 max=85865 max_rank=0 min=6992 min_rank=5 imbalance_ratio=2.620392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[71228, 22082, 45320, 2291, 35931, 37980, 8575, 38737] avg=32768.0000 max=71228 max_rank=0 min=2291 min_rank=3 imbalance_ratio=2.173706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[71228, 22082, 45320, 2291, 35931, 37980, 8575, 38737] avg=32768.0000 max=71228 max_rank=0 min=2291 min_rank=3 imbalance_ratio=2.173706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[71228, 22082, 45320, 2291, 35931, 37980, 8575, 38737] avg=32768.0000 max=71228 max_rank=0 min=2291 min_rank=3 imbalance_ratio=2.173706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[33096, 30915, 32655, 33096, 33097, 33095, 33095, 33095] avg=32768.0000 max=33097 max_rank=4 min=30915 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50271 max_load_reduction_pct=0.603001 moved_to_replicas=111574 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[7953, 50341, 31876, 45242, 48802, 12294, 3327, 62309] avg=32768.0000 max=62309 max_rank=7 min=3327 min_rank=6 imbalance_ratio=1.901520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] avg=32768.0000 max=33096 max_rank=0 min=30961 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38132 max_load_reduction_pct=0.535351 moved_to_replicas=63716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52769 max_load_reduction_pct=0.614558 moved_to_replicas=85251 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[16388, 17915, 63779, 23678, 40558, 58351, 10913, 30562] avg=32768.0000 max=63779 max_rank=2 min=10913 min_rank=6 imbalance_ratio=1.946381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] avg=32768.0000 max=33096 max_rank=0 min=30961 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38132 max_load_reduction_pct=0.535351 moved_to_replicas=63716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] avg=32768.0000 max=33096 max_rank=0 min=30961 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38132 max_load_reduction_pct=0.535351 moved_to_replicas=63716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] avg=32768.0000 max=33096 max_rank=0 min=30961 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38132 max_load_reduction_pct=0.535351 moved_to_replicas=63716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] avg=32768.0000 max=33096 max_rank=0 min=31876 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29213 max_load_reduction_pct=0.468841 moved_to_replicas=74310 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] avg=32768.0000 max=33097 max_rank=2 min=30562 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30682 max_load_reduction_pct=0.481067 moved_to_replicas=63400 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[85865, 17501, 35545, 22706, 23216, 6992, 7190, 63129] avg=32768.0000 max=85865 max_rank=0 min=6992 min_rank=5 imbalance_ratio=2.620392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[7953, 50341, 31876, 45242, 48802, 12294, 3327, 62309] avg=32768.0000 max=62309 max_rank=7 min=3327 min_rank=6 imbalance_ratio=1.901520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[71228, 22082, 45320, 2291, 35931, 37980, 8575, 38737] avg=32768.0000 max=71228 max_rank=0 min=2291 min_rank=3 imbalance_ratio=2.173706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[7953, 50341, 31876, 45242, 48802, 12294, 3327, 62309] avg=32768.0000 max=62309 max_rank=7 min=3327 min_rank=6 imbalance_ratio=1.901520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[7953, 50341, 31876, 45242, 48802, 12294, 3327, 62309] avg=32768.0000 max=62309 max_rank=7 min=3327 min_rank=6 imbalance_ratio=1.901520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] avg=32768.0000 max=33096 max_rank=0 min=31876 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29213 max_load_reduction_pct=0.468841 moved_to_replicas=74310 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[7953, 50341, 31876, 45242, 48802, 12294, 3327, 62309] avg=32768.0000 max=62309 max_rank=7 min=3327 min_rank=6 imbalance_ratio=1.901520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52769 max_load_reduction_pct=0.614558 moved_to_replicas=85251 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[16388, 17915, 63779, 23678, 40558, 58351, 10913, 30562] avg=32768.0000 max=63779 max_rank=2 min=10913 min_rank=6 imbalance_ratio=1.946381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] avg=32768.0000 max=33096 max_rank=0 min=30961 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38132 max_load_reduction_pct=0.535351 moved_to_replicas=63716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[64902, 12628, 33633, 32563, 1929, 34874, 62324, 19291] avg=32768.0000 max=64902 max_rank=0 min=1929 min_rank=4 imbalance_ratio=1.980652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] avg=32768.0000 max=33096 max_rank=0 min=31876 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29213 max_load_reduction_pct=0.468841 moved_to_replicas=74310 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] avg=32768.0000 max=33096 max_rank=0 min=31876 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29213 max_load_reduction_pct=0.468841 moved_to_replicas=74310 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] avg=32768.0000 max=33096 max_rank=0 min=31876 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29213 max_load_reduction_pct=0.468841 moved_to_replicas=74310 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] avg=32768.0000 max=33097 max_rank=2 min=30562 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30682 max_load_reduction_pct=0.481067 moved_to_replicas=63400 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] avg=32768.0000 max=33096 max_rank=0 min=31006 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31806 max_load_reduction_pct=0.490062 moved_to_replicas=63349 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[16388, 17915, 63779, 23678, 40558, 58351, 10913, 30562] avg=32768.0000 max=63779 max_rank=2 min=10913 min_rank=6 imbalance_ratio=1.946381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[71228, 22082, 45320, 2291, 35931, 37980, 8575, 38737] avg=32768.0000 max=71228 max_rank=0 min=2291 min_rank=3 imbalance_ratio=2.173706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[7953, 50341, 31876, 45242, 48802, 12294, 3327, 62309] avg=32768.0000 max=62309 max_rank=7 min=3327 min_rank=6 imbalance_ratio=1.901520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[16388, 17915, 63779, 23678, 40558, 58351, 10913, 30562] avg=32768.0000 max=63779 max_rank=2 min=10913 min_rank=6 imbalance_ratio=1.946381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[16388, 17915, 63779, 23678, 40558, 58351, 10913, 30562] avg=32768.0000 max=63779 max_rank=2 min=10913 min_rank=6 imbalance_ratio=1.946381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] avg=32768.0000 max=33097 max_rank=2 min=30562 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30682 max_load_reduction_pct=0.481067 moved_to_replicas=63400 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[16388, 17915, 63779, 23678, 40558, 58351, 10913, 30562] avg=32768.0000 max=63779 max_rank=2 min=10913 min_rank=6 imbalance_ratio=1.946381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[33096, 32608, 33096, 33096, 33096, 33096, 30961, 33095] avg=32768.0000 max=33096 max_rank=0 min=30961 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=38132 max_load_reduction_pct=0.535351 moved_to_replicas=63716 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[35072, 13539, 51013, 56356, 15512, 32768, 24500, 33384] avg=32768.0000 max=56356 max_rank=3 min=13539 min_rank=1 imbalance_ratio=1.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[64902, 12628, 33633, 32563, 1929, 34874, 62324, 19291] avg=32768.0000 max=64902 max_rank=0 min=1929 min_rank=4 imbalance_ratio=1.980652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] avg=32768.0000 max=33096 max_rank=0 min=31876 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29213 max_load_reduction_pct=0.468841 moved_to_replicas=74310 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] avg=32768.0000 max=33097 max_rank=2 min=30562 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30682 max_load_reduction_pct=0.481067 moved_to_replicas=63400 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] avg=32768.0000 max=33097 max_rank=2 min=30562 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30682 max_load_reduction_pct=0.481067 moved_to_replicas=63400 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] avg=32768.0000 max=33097 max_rank=2 min=30562 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30682 max_load_reduction_pct=0.481067 moved_to_replicas=63400 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] avg=32768.0000 max=33097 max_rank=3 min=30800 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23259 max_load_reduction_pct=0.412716 moved_to_replicas=43441 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] avg=32768.0000 max=33096 max_rank=0 min=31006 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31806 max_load_reduction_pct=0.490062 moved_to_replicas=63349 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[64902, 12628, 33633, 32563, 1929, 34874, 62324, 19291] avg=32768.0000 max=64902 max_rank=0 min=1929 min_rank=4 imbalance_ratio=1.980652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[7953, 50341, 31876, 45242, 48802, 12294, 3327, 62309] avg=32768.0000 max=62309 max_rank=7 min=3327 min_rank=6 imbalance_ratio=1.901520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[16388, 17915, 63779, 23678, 40558, 58351, 10913, 30562] avg=32768.0000 max=63779 max_rank=2 min=10913 min_rank=6 imbalance_ratio=1.946381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[64902, 12628, 33633, 32563, 1929, 34874, 62324, 19291] avg=32768.0000 max=64902 max_rank=0 min=1929 min_rank=4 imbalance_ratio=1.980652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[64902, 12628, 33633, 32563, 1929, 34874, 62324, 19291] avg=32768.0000 max=64902 max_rank=0 min=1929 min_rank=4 imbalance_ratio=1.980652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] avg=32768.0000 max=33096 max_rank=0 min=31006 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31806 max_load_reduction_pct=0.490062 moved_to_replicas=63349 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[33905, 5024, 317, 27310, 84686, 30574, 234, 80094] avg=32768.0000 max=84686 max_rank=4 min=234 min_rank=6 imbalance_ratio=2.584412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[64902, 12628, 33633, 32563, 1929, 34874, 62324, 19291] avg=32768.0000 max=64902 max_rank=0 min=1929 min_rank=4 imbalance_ratio=1.980652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[35072, 13539, 51013, 56356, 15512, 32768, 24500, 33384] avg=32768.0000 max=56356 max_rank=3 min=13539 min_rank=1 imbalance_ratio=1.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[33096, 33096, 31876, 33096, 33094, 32250, 32540, 33096] avg=32768.0000 max=33096 max_rank=0 min=31876 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29213 max_load_reduction_pct=0.468841 moved_to_replicas=74310 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] avg=32768.0000 max=33097 max_rank=2 min=30562 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30682 max_load_reduction_pct=0.481067 moved_to_replicas=63400 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] avg=32768.0000 max=33096 max_rank=0 min=31006 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31806 max_load_reduction_pct=0.490062 moved_to_replicas=63349 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] avg=32768.0000 max=33096 max_rank=0 min=31006 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31806 max_load_reduction_pct=0.490062 moved_to_replicas=63349 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31541 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=51589 max_load_reduction_pct=0.609180 moved_to_replicas=99397 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] avg=32768.0000 max=33096 max_rank=0 min=31006 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31806 max_load_reduction_pct=0.490062 moved_to_replicas=63349 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] avg=32768.0000 max=33097 max_rank=3 min=30800 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23259 max_load_reduction_pct=0.412716 moved_to_replicas=43441 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[35072, 13539, 51013, 56356, 15512, 32768, 24500, 33384] avg=32768.0000 max=56356 max_rank=3 min=13539 min_rank=1 imbalance_ratio=1.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[16388, 17915, 63779, 23678, 40558, 58351, 10913, 30562] avg=32768.0000 max=63779 max_rank=2 min=10913 min_rank=6 imbalance_ratio=1.946381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[64902, 12628, 33633, 32563, 1929, 34874, 62324, 19291] avg=32768.0000 max=64902 max_rank=0 min=1929 min_rank=4 imbalance_ratio=1.980652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[35072, 13539, 51013, 56356, 15512, 32768, 24500, 33384] avg=32768.0000 max=56356 max_rank=3 min=13539 min_rank=1 imbalance_ratio=1.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[35072, 13539, 51013, 56356, 15512, 32768, 24500, 33384] avg=32768.0000 max=56356 max_rank=3 min=13539 min_rank=1 imbalance_ratio=1.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] avg=32768.0000 max=33097 max_rank=3 min=30800 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23259 max_load_reduction_pct=0.412716 moved_to_replicas=43441 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[7149, 3799, 51962, 3076, 60567, 44740, 75378, 15473] avg=32768.0000 max=75378 max_rank=6 min=3076 min_rank=3 imbalance_ratio=2.300354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[33905, 5024, 317, 27310, 84686, 30574, 234, 80094] avg=32768.0000 max=84686 max_rank=4 min=234 min_rank=6 imbalance_ratio=2.584412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] avg=32768.0000 max=33096 max_rank=0 min=31006 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31806 max_load_reduction_pct=0.490062 moved_to_replicas=63349 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[35072, 13539, 51013, 56356, 15512, 32768, 24500, 33384] avg=32768.0000 max=56356 max_rank=3 min=13539 min_rank=1 imbalance_ratio=1.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33095, 33097, 33006, 33096, 33097, 33096, 30562] avg=32768.0000 max=33097 max_rank=2 min=30562 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=30682 max_load_reduction_pct=0.481067 moved_to_replicas=63400 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] avg=32768.0000 max=33097 max_rank=3 min=30800 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23259 max_load_reduction_pct=0.412716 moved_to_replicas=43441 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] avg=32768.0000 max=33097 max_rank=3 min=30800 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23259 max_load_reduction_pct=0.412716 moved_to_replicas=43441 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] avg=32768.0000 max=33096 max_rank=0 min=30635 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42282 max_load_reduction_pct=0.560933 moved_to_replicas=100263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31541 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=51589 max_load_reduction_pct=0.609180 moved_to_replicas=99397 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] avg=32768.0000 max=33097 max_rank=3 min=30800 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23259 max_load_reduction_pct=0.412716 moved_to_replicas=43441 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[33905, 5024, 317, 27310, 84686, 30574, 234, 80094] avg=32768.0000 max=84686 max_rank=4 min=234 min_rank=6 imbalance_ratio=2.584412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[35072, 13539, 51013, 56356, 15512, 32768, 24500, 33384] avg=32768.0000 max=56356 max_rank=3 min=13539 min_rank=1 imbalance_ratio=1.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[33905, 5024, 317, 27310, 84686, 30574, 234, 80094] avg=32768.0000 max=84686 max_rank=4 min=234 min_rank=6 imbalance_ratio=2.584412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[64902, 12628, 33633, 32563, 1929, 34874, 62324, 19291] avg=32768.0000 max=64902 max_rank=0 min=1929 min_rank=4 imbalance_ratio=1.980652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[33905, 5024, 317, 27310, 84686, 30574, 234, 80094] avg=32768.0000 max=84686 max_rank=4 min=234 min_rank=6 imbalance_ratio=2.584412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[35114, 34719, 24219, 12429, 23732, 55655, 43667, 32609] avg=32768.0000 max=55655 max_rank=5 min=12429 min_rank=3 imbalance_ratio=1.698456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31541 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=51589 max_load_reduction_pct=0.609180 moved_to_replicas=99397 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[7149, 3799, 51962, 3076, 60567, 44740, 75378, 15473] avg=32768.0000 max=75378 max_rank=6 min=3076 min_rank=3 imbalance_ratio=2.300354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] avg=32768.0000 max=33097 max_rank=3 min=30800 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23259 max_load_reduction_pct=0.412716 moved_to_replicas=43441 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[33905, 5024, 317, 27310, 84686, 30574, 234, 80094] avg=32768.0000 max=84686 max_rank=4 min=234 min_rank=6 imbalance_ratio=2.584412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31541 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=51589 max_load_reduction_pct=0.609180 moved_to_replicas=99397 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31541 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=51589 max_load_reduction_pct=0.609180 moved_to_replicas=99397 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[33096, 33095, 33096, 32563, 33096, 33096, 33096, 31006] avg=32768.0000 max=33096 max_rank=0 min=31006 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31806 max_load_reduction_pct=0.490062 moved_to_replicas=63349 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] avg=32768.0000 max=33096 max_rank=0 min=30959 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=22559 max_load_reduction_pct=0.405336 moved_to_replicas=36771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] avg=32768.0000 max=33096 max_rank=0 min=30635 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42282 max_load_reduction_pct=0.560933 moved_to_replicas=100263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31541 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=51589 max_load_reduction_pct=0.609180 moved_to_replicas=99397 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[7149, 3799, 51962, 3076, 60567, 44740, 75378, 15473] avg=32768.0000 max=75378 max_rank=6 min=3076 min_rank=3 imbalance_ratio=2.300354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[33905, 5024, 317, 27310, 84686, 30574, 234, 80094] avg=32768.0000 max=84686 max_rank=4 min=234 min_rank=6 imbalance_ratio=2.584412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[7149, 3799, 51962, 3076, 60567, 44740, 75378, 15473] avg=32768.0000 max=75378 max_rank=6 min=3076 min_rank=3 imbalance_ratio=2.300354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[7149, 3799, 51962, 3076, 60567, 44740, 75378, 15473] avg=32768.0000 max=75378 max_rank=6 min=3076 min_rank=3 imbalance_ratio=2.300354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[74863, 23422, 66506, 26744, 3837, 27653, 40, 39079] avg=32768.0000 max=74863 max_rank=0 min=40 min_rank=6 imbalance_ratio=2.284637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[35072, 13539, 51013, 56356, 15512, 32768, 24500, 33384] avg=32768.0000 max=56356 max_rank=3 min=13539 min_rank=1 imbalance_ratio=1.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[35114, 34719, 24219, 12429, 23732, 55655, 43667, 32609] avg=32768.0000 max=55655 max_rank=5 min=12429 min_rank=3 imbalance_ratio=1.698456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] avg=32768.0000 max=33096 max_rank=0 min=30635 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42282 max_load_reduction_pct=0.560933 moved_to_replicas=100263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[7149, 3799, 51962, 3076, 60567, 44740, 75378, 15473] avg=32768.0000 max=75378 max_rank=6 min=3076 min_rank=3 imbalance_ratio=2.300354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] avg=32768.0000 max=33096 max_rank=0 min=30635 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42282 max_load_reduction_pct=0.560933 moved_to_replicas=100263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31541 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=51589 max_load_reduction_pct=0.609180 moved_to_replicas=99397 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] avg=32768.0000 max=33098 max_rank=7 min=30896 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41765 max_load_reduction_pct=0.557886 moved_to_replicas=81160 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] avg=32768.0000 max=33096 max_rank=0 min=30635 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42282 max_load_reduction_pct=0.560933 moved_to_replicas=100263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 32768, 30800, 33096] avg=32768.0000 max=33097 max_rank=3 min=30800 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23259 max_load_reduction_pct=0.412716 moved_to_replicas=43441 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] avg=32768.0000 max=33096 max_rank=0 min=30959 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=22559 max_load_reduction_pct=0.405336 moved_to_replicas=36771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] avg=32768.0000 max=33096 max_rank=0 min=30635 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42282 max_load_reduction_pct=0.560933 moved_to_replicas=100263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[35114, 34719, 24219, 12429, 23732, 55655, 43667, 32609] avg=32768.0000 max=55655 max_rank=5 min=12429 min_rank=3 imbalance_ratio=1.698456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[35114, 34719, 24219, 12429, 23732, 55655, 43667, 32609] avg=32768.0000 max=55655 max_rank=5 min=12429 min_rank=3 imbalance_ratio=1.698456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[7149, 3799, 51962, 3076, 60567, 44740, 75378, 15473] avg=32768.0000 max=75378 max_rank=6 min=3076 min_rank=3 imbalance_ratio=2.300354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[35530, 2673, 70691, 49556, 322, 43029, 16546, 43797] avg=32768.0000 max=70691 max_rank=2 min=322 min_rank=4 imbalance_ratio=2.157318 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[35114, 34719, 24219, 12429, 23732, 55655, 43667, 32609] avg=32768.0000 max=55655 max_rank=5 min=12429 min_rank=3 imbalance_ratio=1.698456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[33905, 5024, 317, 27310, 84686, 30574, 234, 80094] avg=32768.0000 max=84686 max_rank=4 min=234 min_rank=6 imbalance_ratio=2.584412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[74863, 23422, 66506, 26744, 3837, 27653, 40, 39079] avg=32768.0000 max=74863 max_rank=0 min=40 min_rank=6 imbalance_ratio=2.284637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] avg=32768.0000 max=33096 max_rank=0 min=30959 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=22559 max_load_reduction_pct=0.405336 moved_to_replicas=36771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[35114, 34719, 24219, 12429, 23732, 55655, 43667, 32609] avg=32768.0000 max=55655 max_rank=5 min=12429 min_rank=3 imbalance_ratio=1.698456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] avg=32768.0000 max=33096 max_rank=0 min=30959 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=22559 max_load_reduction_pct=0.405336 moved_to_replicas=36771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] avg=32768.0000 max=33096 max_rank=0 min=30635 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42282 max_load_reduction_pct=0.560933 moved_to_replicas=100263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] avg=32768.0000 max=33097 max_rank=3 min=30661 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37594 max_load_reduction_pct=0.531807 moved_to_replicas=77123 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] avg=32768.0000 max=33096 max_rank=0 min=30959 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=22559 max_load_reduction_pct=0.405336 moved_to_replicas=36771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 32027, 33096, 33096, 31541, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31541 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=51589 max_load_reduction_pct=0.609180 moved_to_replicas=99397 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] avg=32768.0000 max=33098 max_rank=7 min=30896 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41765 max_load_reduction_pct=0.557886 moved_to_replicas=81160 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] avg=32768.0000 max=33096 max_rank=0 min=30959 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=22559 max_load_reduction_pct=0.405336 moved_to_replicas=36771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[74863, 23422, 66506, 26744, 3837, 27653, 40, 39079] avg=32768.0000 max=74863 max_rank=0 min=40 min_rank=6 imbalance_ratio=2.284637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[74863, 23422, 66506, 26744, 3837, 27653, 40, 39079] avg=32768.0000 max=74863 max_rank=0 min=40 min_rank=6 imbalance_ratio=2.284637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[35114, 34719, 24219, 12429, 23732, 55655, 43667, 32609] avg=32768.0000 max=55655 max_rank=5 min=12429 min_rank=3 imbalance_ratio=1.698456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[18270, 34022, 31048, 42647, 61405, 5926, 36286, 32540] avg=32768.0000 max=61405 max_rank=4 min=5926 min_rank=5 imbalance_ratio=1.873932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[74863, 23422, 66506, 26744, 3837, 27653, 40, 39079] avg=32768.0000 max=74863 max_rank=0 min=40 min_rank=6 imbalance_ratio=2.284637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[35530, 2673, 70691, 49556, 322, 43029, 16546, 43797] avg=32768.0000 max=70691 max_rank=2 min=322 min_rank=4 imbalance_ratio=2.157318 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[7149, 3799, 51962, 3076, 60567, 44740, 75378, 15473] avg=32768.0000 max=75378 max_rank=6 min=3076 min_rank=3 imbalance_ratio=2.300354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] avg=32768.0000 max=33098 max_rank=7 min=30896 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41765 max_load_reduction_pct=0.557886 moved_to_replicas=81160 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[74863, 23422, 66506, 26744, 3837, 27653, 40, 39079] avg=32768.0000 max=74863 max_rank=0 min=40 min_rank=6 imbalance_ratio=2.284637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] avg=32768.0000 max=33098 max_rank=7 min=30896 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41765 max_load_reduction_pct=0.557886 moved_to_replicas=81160 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] avg=32768.0000 max=33096 max_rank=0 min=30959 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=22559 max_load_reduction_pct=0.405336 moved_to_replicas=36771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] avg=32768.0000 max=33097 max_rank=3 min=31974 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28308 max_load_reduction_pct=0.461005 moved_to_replicas=41976 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] avg=32768.0000 max=33098 max_rank=7 min=30896 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41765 max_load_reduction_pct=0.557886 moved_to_replicas=81160 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] avg=32768.0000 max=33097 max_rank=3 min=30661 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37594 max_load_reduction_pct=0.531807 moved_to_replicas=77123 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[33096, 33096, 33096, 32933, 33096, 33096, 33096, 30635] avg=32768.0000 max=33096 max_rank=0 min=30635 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42282 max_load_reduction_pct=0.560933 moved_to_replicas=100263 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] avg=32768.0000 max=33098 max_rank=7 min=30896 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41765 max_load_reduction_pct=0.557886 moved_to_replicas=81160 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[35530, 2673, 70691, 49556, 322, 43029, 16546, 43797] avg=32768.0000 max=70691 max_rank=2 min=322 min_rank=4 imbalance_ratio=2.157318 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[35530, 2673, 70691, 49556, 322, 43029, 16546, 43797] avg=32768.0000 max=70691 max_rank=2 min=322 min_rank=4 imbalance_ratio=2.157318 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[3511, 61665, 3928, 32199, 31713, 33896, 35360, 59872] avg=32768.0000 max=61665 max_rank=1 min=3511 min_rank=0 imbalance_ratio=1.881866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[74863, 23422, 66506, 26744, 3837, 27653, 40, 39079] avg=32768.0000 max=74863 max_rank=0 min=40 min_rank=6 imbalance_ratio=2.284637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[35530, 2673, 70691, 49556, 322, 43029, 16546, 43797] avg=32768.0000 max=70691 max_rank=2 min=322 min_rank=4 imbalance_ratio=2.157318 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[18270, 34022, 31048, 42647, 61405, 5926, 36286, 32540] avg=32768.0000 max=61405 max_rank=4 min=5926 min_rank=5 imbalance_ratio=1.873932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[35114, 34719, 24219, 12429, 23732, 55655, 43667, 32609] avg=32768.0000 max=55655 max_rank=5 min=12429 min_rank=3 imbalance_ratio=1.698456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] avg=32768.0000 max=33097 max_rank=3 min=30661 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37594 max_load_reduction_pct=0.531807 moved_to_replicas=77123 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] avg=32768.0000 max=33097 max_rank=3 min=30661 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37594 max_load_reduction_pct=0.531807 moved_to_replicas=77123 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[35530, 2673, 70691, 49556, 322, 43029, 16546, 43797] avg=32768.0000 max=70691 max_rank=2 min=322 min_rank=4 imbalance_ratio=2.157318 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32080 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28569 max_load_reduction_pct=0.463294 moved_to_replicas=58409 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] avg=32768.0000 max=33098 max_rank=7 min=30896 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41765 max_load_reduction_pct=0.557886 moved_to_replicas=81160 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] avg=32768.0000 max=33097 max_rank=3 min=30661 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37594 max_load_reduction_pct=0.531807 moved_to_replicas=77123 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30959, 33096, 33096, 32609] avg=32768.0000 max=33096 max_rank=0 min=30959 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=22559 max_load_reduction_pct=0.405336 moved_to_replicas=36771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] avg=32768.0000 max=33097 max_rank=3 min=31974 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28308 max_load_reduction_pct=0.461005 moved_to_replicas=41976 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] avg=32768.0000 max=33097 max_rank=3 min=30661 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37594 max_load_reduction_pct=0.531807 moved_to_replicas=77123 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[18270, 34022, 31048, 42647, 61405, 5926, 36286, 32540] avg=32768.0000 max=61405 max_rank=4 min=5926 min_rank=5 imbalance_ratio=1.873932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[18270, 34022, 31048, 42647, 61405, 5926, 36286, 32540] avg=32768.0000 max=61405 max_rank=4 min=5926 min_rank=5 imbalance_ratio=1.873932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[39911, 736, 117646, 40356, 8510, 27979, 7160, 19846] avg=32768.0000 max=117646 max_rank=2 min=736 min_rank=1 imbalance_ratio=3.590271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[35530, 2673, 70691, 49556, 322, 43029, 16546, 43797] avg=32768.0000 max=70691 max_rank=2 min=322 min_rank=4 imbalance_ratio=2.157318 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[18270, 34022, 31048, 42647, 61405, 5926, 36286, 32540] avg=32768.0000 max=61405 max_rank=4 min=5926 min_rank=5 imbalance_ratio=1.873932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[3511, 61665, 3928, 32199, 31713, 33896, 35360, 59872] avg=32768.0000 max=61665 max_rank=1 min=3511 min_rank=0 imbalance_ratio=1.881866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] avg=32768.0000 max=33097 max_rank=3 min=31974 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28308 max_load_reduction_pct=0.461005 moved_to_replicas=41976 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[74863, 23422, 66506, 26744, 3837, 27653, 40, 39079] avg=32768.0000 max=74863 max_rank=0 min=40 min_rank=6 imbalance_ratio=2.284637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] avg=32768.0000 max=33097 max_rank=3 min=31974 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28308 max_load_reduction_pct=0.461005 moved_to_replicas=41976 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[18270, 34022, 31048, 42647, 61405, 5926, 36286, 32540] avg=32768.0000 max=61405 max_rank=4 min=5926 min_rank=5 imbalance_ratio=1.873932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] avg=32768.0000 max=33098 max_rank=6 min=31626 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=84548 max_load_reduction_pct=0.718664 moved_to_replicas=98625 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] avg=32768.0000 max=33097 max_rank=3 min=30661 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37594 max_load_reduction_pct=0.531807 moved_to_replicas=77123 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] avg=32768.0000 max=33097 max_rank=3 min=31974 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28308 max_load_reduction_pct=0.461005 moved_to_replicas=41976 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32080 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28569 max_load_reduction_pct=0.463294 moved_to_replicas=58409 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[33097, 33096, 33096, 30896, 33095, 33094, 32672, 33098] avg=32768.0000 max=33098 max_rank=7 min=30896 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=41765 max_load_reduction_pct=0.557886 moved_to_replicas=81160 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] avg=32768.0000 max=33097 max_rank=3 min=31974 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28308 max_load_reduction_pct=0.461005 moved_to_replicas=41976 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[3511, 61665, 3928, 32199, 31713, 33896, 35360, 59872] avg=32768.0000 max=61665 max_rank=1 min=3511 min_rank=0 imbalance_ratio=1.881866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[3511, 61665, 3928, 32199, 31713, 33896, 35360, 59872] avg=32768.0000 max=61665 max_rank=1 min=3511 min_rank=0 imbalance_ratio=1.881866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[31577, 60325, 36086, 16718, 196, 55021, 4589, 57632] avg=32768.0000 max=60325 max_rank=1 min=196 min_rank=4 imbalance_ratio=1.840973 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32080 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28569 max_load_reduction_pct=0.463294 moved_to_replicas=58409 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[3511, 61665, 3928, 32199, 31713, 33896, 35360, 59872] avg=32768.0000 max=61665 max_rank=1 min=3511 min_rank=0 imbalance_ratio=1.881866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32080 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28569 max_load_reduction_pct=0.463294 moved_to_replicas=58409 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[18270, 34022, 31048, 42647, 61405, 5926, 36286, 32540] avg=32768.0000 max=61405 max_rank=4 min=5926 min_rank=5 imbalance_ratio=1.873932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[39911, 736, 117646, 40356, 8510, 27979, 7160, 19846] avg=32768.0000 max=117646 max_rank=2 min=736 min_rank=1 imbalance_ratio=3.590271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] avg=32768.0000 max=33097 max_rank=3 min=31577 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27228 max_load_reduction_pct=0.451355 moved_to_replicas=76680 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[35530, 2673, 70691, 49556, 322, 43029, 16546, 43797] avg=32768.0000 max=70691 max_rank=2 min=322 min_rank=4 imbalance_ratio=2.157318 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[3511, 61665, 3928, 32199, 31713, 33896, 35360, 59872] avg=32768.0000 max=61665 max_rank=1 min=3511 min_rank=0 imbalance_ratio=1.881866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32080 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28569 max_load_reduction_pct=0.463294 moved_to_replicas=58409 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] avg=32768.0000 max=33098 max_rank=6 min=31626 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=84548 max_load_reduction_pct=0.718664 moved_to_replicas=98625 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] avg=32768.0000 max=33097 max_rank=3 min=31974 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28308 max_load_reduction_pct=0.461005 moved_to_replicas=41976 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[39911, 736, 117646, 40356, 8510, 27979, 7160, 19846] avg=32768.0000 max=117646 max_rank=2 min=736 min_rank=1 imbalance_ratio=3.590271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 32907, 33096, 30661, 33096] avg=32768.0000 max=33097 max_rank=3 min=30661 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=37594 max_load_reduction_pct=0.531807 moved_to_replicas=77123 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32080 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28569 max_load_reduction_pct=0.463294 moved_to_replicas=58409 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[39911, 736, 117646, 40356, 8510, 27979, 7160, 19846] avg=32768.0000 max=117646 max_rank=2 min=736 min_rank=1 imbalance_ratio=3.590271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[2813, 66904, 12906, 75630, 24375, 39466, 39717, 333] avg=32768.0000 max=75630 max_rank=3 min=333 min_rank=7 imbalance_ratio=2.308044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] avg=32768.0000 max=33098 max_rank=6 min=31626 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=84548 max_load_reduction_pct=0.718664 moved_to_replicas=98625 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[39911, 736, 117646, 40356, 8510, 27979, 7160, 19846] avg=32768.0000 max=117646 max_rank=2 min=736 min_rank=1 imbalance_ratio=3.590271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] avg=32768.0000 max=33098 max_rank=6 min=31626 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=84548 max_load_reduction_pct=0.718664 moved_to_replicas=98625 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[31577, 60325, 36086, 16718, 196, 55021, 4589, 57632] avg=32768.0000 max=60325 max_rank=1 min=196 min_rank=4 imbalance_ratio=1.840973 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[3511, 61665, 3928, 32199, 31713, 33896, 35360, 59872] avg=32768.0000 max=61665 max_rank=1 min=3511 min_rank=0 imbalance_ratio=1.881866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] avg=32768.0000 max=33097 max_rank=6 min=31912 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=42533 max_load_reduction_pct=0.562383 moved_to_replicas=89333 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[18270, 34022, 31048, 42647, 61405, 5926, 36286, 32540] avg=32768.0000 max=61405 max_rank=4 min=5926 min_rank=5 imbalance_ratio=1.873932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[39911, 736, 117646, 40356, 8510, 27979, 7160, 19846] avg=32768.0000 max=117646 max_rank=2 min=736 min_rank=1 imbalance_ratio=3.590271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] avg=32768.0000 max=33098 max_rank=6 min=31626 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=84548 max_load_reduction_pct=0.718664 moved_to_replicas=98625 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] avg=32768.0000 max=33097 max_rank=3 min=31577 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27228 max_load_reduction_pct=0.451355 moved_to_replicas=76680 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[31577, 60325, 36086, 16718, 196, 55021, 4589, 57632] avg=32768.0000 max=60325 max_rank=1 min=196 min_rank=4 imbalance_ratio=1.840973 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32080 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28569 max_load_reduction_pct=0.463294 moved_to_replicas=58409 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[32149, 33096, 31974, 33097, 33095, 33097, 33096, 32540] avg=32768.0000 max=33097 max_rank=3 min=31974 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=28308 max_load_reduction_pct=0.461005 moved_to_replicas=41976 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] avg=32768.0000 max=33098 max_rank=6 min=31626 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=84548 max_load_reduction_pct=0.718664 moved_to_replicas=98625 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[31577, 60325, 36086, 16718, 196, 55021, 4589, 57632] avg=32768.0000 max=60325 max_rank=1 min=196 min_rank=4 imbalance_ratio=1.840973 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[53969, 20849, 43572, 45289, 30906, 54265, 862, 12432] avg=32768.0000 max=54265 max_rank=5 min=862 min_rank=6 imbalance_ratio=1.656036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] avg=32768.0000 max=33097 max_rank=3 min=31577 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27228 max_load_reduction_pct=0.451355 moved_to_replicas=76680 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[2813, 66904, 12906, 75630, 24375, 39466, 39717, 333] avg=32768.0000 max=75630 max_rank=3 min=333 min_rank=7 imbalance_ratio=2.308044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] avg=32768.0000 max=33098 max_rank=5 min=30906 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21167 max_load_reduction_pct=0.390067 moved_to_replicas=64711 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[31577, 60325, 36086, 16718, 196, 55021, 4589, 57632] avg=32768.0000 max=60325 max_rank=1 min=196 min_rank=4 imbalance_ratio=1.840973 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] avg=32768.0000 max=33097 max_rank=3 min=31577 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27228 max_load_reduction_pct=0.451355 moved_to_replicas=76680 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[39911, 736, 117646, 40356, 8510, 27979, 7160, 19846] avg=32768.0000 max=117646 max_rank=2 min=736 min_rank=1 imbalance_ratio=3.590271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[3511, 61665, 3928, 32199, 31713, 33896, 35360, 59872] avg=32768.0000 max=61665 max_rank=1 min=3511 min_rank=0 imbalance_ratio=1.881866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[31577, 60325, 36086, 16718, 196, 55021, 4589, 57632] avg=32768.0000 max=60325 max_rank=1 min=196 min_rank=4 imbalance_ratio=1.840973 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] avg=32768.0000 max=33097 max_rank=6 min=31912 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=42533 max_load_reduction_pct=0.562383 moved_to_replicas=89333 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[2813, 66904, 12906, 75630, 24375, 39466, 39717, 333] avg=32768.0000 max=75630 max_rank=3 min=333 min_rank=7 imbalance_ratio=2.308044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] avg=32768.0000 max=33097 max_rank=3 min=31577 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27228 max_load_reduction_pct=0.451355 moved_to_replicas=76680 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] avg=32768.0000 max=33098 max_rank=6 min=31626 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=84548 max_load_reduction_pct=0.718664 moved_to_replicas=98625 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[32080, 33096, 32968, 32199, 32513, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32080 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28569 max_load_reduction_pct=0.463294 moved_to_replicas=58409 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[31020, 32780, 30327, 27760, 325, 75641, 23558, 40733] avg=32768.0000 max=75641 max_rank=5 min=325 min_rank=4 imbalance_ratio=2.308380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] avg=32768.0000 max=33097 max_rank=3 min=31577 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27228 max_load_reduction_pct=0.451355 moved_to_replicas=76680 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[2813, 66904, 12906, 75630, 24375, 39466, 39717, 333] avg=32768.0000 max=75630 max_rank=3 min=333 min_rank=7 imbalance_ratio=2.308044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] avg=32768.0000 max=33097 max_rank=6 min=31912 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=42533 max_load_reduction_pct=0.562383 moved_to_replicas=89333 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[53969, 20849, 43572, 45289, 30906, 54265, 862, 12432] avg=32768.0000 max=54265 max_rank=5 min=862 min_rank=6 imbalance_ratio=1.656036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31493 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42545 max_load_reduction_pct=0.562460 moved_to_replicas=50182 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] avg=32768.0000 max=33097 max_rank=6 min=31912 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=42533 max_load_reduction_pct=0.562383 moved_to_replicas=89333 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[2813, 66904, 12906, 75630, 24375, 39466, 39717, 333] avg=32768.0000 max=75630 max_rank=3 min=333 min_rank=7 imbalance_ratio=2.308044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[31577, 60325, 36086, 16718, 196, 55021, 4589, 57632] avg=32768.0000 max=60325 max_rank=1 min=196 min_rank=4 imbalance_ratio=1.840973 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[39911, 736, 117646, 40356, 8510, 27979, 7160, 19846] avg=32768.0000 max=117646 max_rank=2 min=736 min_rank=1 imbalance_ratio=3.590271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[2813, 66904, 12906, 75630, 24375, 39466, 39717, 333] avg=32768.0000 max=75630 max_rank=3 min=333 min_rank=7 imbalance_ratio=2.308044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] avg=32768.0000 max=33098 max_rank=5 min=30906 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21167 max_load_reduction_pct=0.390067 moved_to_replicas=64711 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[53969, 20849, 43572, 45289, 30906, 54265, 862, 12432] avg=32768.0000 max=54265 max_rank=5 min=862 min_rank=6 imbalance_ratio=1.656036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] avg=32768.0000 max=33097 max_rank=6 min=31912 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=42533 max_load_reduction_pct=0.562383 moved_to_replicas=89333 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] avg=32768.0000 max=33097 max_rank=3 min=31577 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27228 max_load_reduction_pct=0.451355 moved_to_replicas=76680 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=25 tokens_per_ep_rank=[6407, 31165, 15561, 12330, 227, 117442, 32787, 46225] avg=32768.0000 max=117442 max_rank=5 min=227 min_rank=4 imbalance_ratio=3.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[53969, 20849, 43572, 45289, 30906, 54265, 862, 12432] avg=32768.0000 max=54265 max_rank=5 min=862 min_rank=6 imbalance_ratio=1.656036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] avg=32768.0000 max=33097 max_rank=6 min=31912 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=42533 max_load_reduction_pct=0.562383 moved_to_replicas=89333 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[33097, 31943, 33094, 33095, 33096, 33095, 33098, 31626] avg=32768.0000 max=33098 max_rank=6 min=31626 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=84548 max_load_reduction_pct=0.718664 moved_to_replicas=98625 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] avg=32768.0000 max=33098 max_rank=5 min=30906 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21167 max_load_reduction_pct=0.390067 moved_to_replicas=64711 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[31020, 32780, 30327, 27760, 325, 75641, 23558, 40733] avg=32768.0000 max=75641 max_rank=5 min=325 min_rank=4 imbalance_ratio=2.308380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=0 min=31165 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=84346 max_load_reduction_pct=0.718193 moved_to_replicas=97475 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] avg=32768.0000 max=33098 max_rank=5 min=30906 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21167 max_load_reduction_pct=0.390067 moved_to_replicas=64711 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[53969, 20849, 43572, 45289, 30906, 54265, 862, 12432] avg=32768.0000 max=54265 max_rank=5 min=862 min_rank=6 imbalance_ratio=1.656036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[2813, 66904, 12906, 75630, 24375, 39466, 39717, 333] avg=32768.0000 max=75630 max_rank=3 min=333 min_rank=7 imbalance_ratio=2.308044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31493 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42545 max_load_reduction_pct=0.562460 moved_to_replicas=50182 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[53969, 20849, 43572, 45289, 30906, 54265, 862, 12432] avg=32768.0000 max=54265 max_rank=5 min=862 min_rank=6 imbalance_ratio=1.656036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[31577, 60325, 36086, 16718, 196, 55021, 4589, 57632] avg=32768.0000 max=60325 max_rank=1 min=196 min_rank=4 imbalance_ratio=1.840973 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[31020, 32780, 30327, 27760, 325, 75641, 23558, 40733] avg=32768.0000 max=75641 max_rank=5 min=325 min_rank=4 imbalance_ratio=2.308380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] avg=32768.0000 max=33098 max_rank=5 min=30906 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21167 max_load_reduction_pct=0.390067 moved_to_replicas=64711 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] avg=32768.0000 max=33097 max_rank=6 min=31912 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=42533 max_load_reduction_pct=0.562383 moved_to_replicas=89333 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=26 tokens_per_ep_rank=[51305, 7457, 32803, 51867, 68662, 18377, 423, 31250] avg=32768.0000 max=68662 max_rank=4 min=423 min_rank=6 imbalance_ratio=2.095398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[31020, 32780, 30327, 27760, 325, 75641, 23558, 40733] avg=32768.0000 max=75641 max_rank=5 min=325 min_rank=4 imbalance_ratio=2.308380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] avg=32768.0000 max=33098 max_rank=5 min=30906 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21167 max_load_reduction_pct=0.390067 moved_to_replicas=64711 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[31577, 33095, 33096, 33097, 32972, 33096, 32114, 33097] avg=32768.0000 max=33097 max_rank=3 min=31577 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27228 max_load_reduction_pct=0.451355 moved_to_replicas=76680 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31493 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42545 max_load_reduction_pct=0.562460 moved_to_replicas=50182 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=25 tokens_per_ep_rank=[6407, 31165, 15561, 12330, 227, 117442, 32787, 46225] avg=32768.0000 max=117442 max_rank=5 min=227 min_rank=4 imbalance_ratio=3.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] avg=32768.0000 max=33097 max_rank=0 min=31250 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35565 max_load_reduction_pct=0.517972 moved_to_replicas=72546 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31493 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42545 max_load_reduction_pct=0.562460 moved_to_replicas=50182 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[31020, 32780, 30327, 27760, 325, 75641, 23558, 40733] avg=32768.0000 max=75641 max_rank=5 min=325 min_rank=4 imbalance_ratio=2.308380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[53969, 20849, 43572, 45289, 30906, 54265, 862, 12432] avg=32768.0000 max=54265 max_rank=5 min=862 min_rank=6 imbalance_ratio=1.656036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=0 min=31165 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=84346 max_load_reduction_pct=0.718193 moved_to_replicas=97475 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[31020, 32780, 30327, 27760, 325, 75641, 23558, 40733] avg=32768.0000 max=75641 max_rank=5 min=325 min_rank=4 imbalance_ratio=2.308380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31493 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42545 max_load_reduction_pct=0.562460 moved_to_replicas=50182 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=25 tokens_per_ep_rank=[6407, 31165, 15561, 12330, 227, 117442, 32787, 46225] avg=32768.0000 max=117442 max_rank=5 min=227 min_rank=4 imbalance_ratio=3.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[2813, 66904, 12906, 75630, 24375, 39466, 39717, 333] avg=32768.0000 max=75630 max_rank=3 min=333 min_rank=7 imbalance_ratio=2.308044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] avg=32768.0000 max=33098 max_rank=5 min=30906 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21167 max_load_reduction_pct=0.390067 moved_to_replicas=64711 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=27 tokens_per_ep_rank=[29039, 29834, 13486, 21777, 38558, 41062, 70631, 17757] avg=32768.0000 max=70631 max_rank=6 min=13486 min_rank=2 imbalance_ratio=2.155487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=25 tokens_per_ep_rank=[6407, 31165, 15561, 12330, 227, 117442, 32787, 46225] avg=32768.0000 max=117442 max_rank=5 min=227 min_rank=4 imbalance_ratio=3.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31493 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42545 max_load_reduction_pct=0.562460 moved_to_replicas=50182 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=26 tokens_per_ep_rank=[51305, 7457, 32803, 51867, 68662, 18377, 423, 31250] avg=32768.0000 max=68662 max_rank=4 min=423 min_rank=6 imbalance_ratio=2.095398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=0 min=31165 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=84346 max_load_reduction_pct=0.718193 moved_to_replicas=97475 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[32059, 33096, 33096, 33096, 31912, 33096, 33097, 32692] avg=32768.0000 max=33097 max_rank=6 min=31912 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=42533 max_load_reduction_pct=0.562383 moved_to_replicas=89333 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=27 tokens_per_ep_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] avg=32768.0000 max=33097 max_rank=6 min=30850 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37534 max_load_reduction_pct=0.531410 moved_to_replicas=50963 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=25 tokens_per_ep_rank=[6407, 31165, 15561, 12330, 227, 117442, 32787, 46225] avg=32768.0000 max=117442 max_rank=5 min=227 min_rank=4 imbalance_ratio=3.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=0 min=31165 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=84346 max_load_reduction_pct=0.718193 moved_to_replicas=97475 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] avg=32768.0000 max=33097 max_rank=0 min=31250 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35565 max_load_reduction_pct=0.517972 moved_to_replicas=72546 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[31020, 32780, 30327, 27760, 325, 75641, 23558, 40733] avg=32768.0000 max=75641 max_rank=5 min=325 min_rank=4 imbalance_ratio=2.308380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=25 tokens_per_ep_rank=[6407, 31165, 15561, 12330, 227, 117442, 32787, 46225] avg=32768.0000 max=117442 max_rank=5 min=227 min_rank=4 imbalance_ratio=3.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=0 min=31165 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=84346 max_load_reduction_pct=0.718193 moved_to_replicas=97475 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=26 tokens_per_ep_rank=[51305, 7457, 32803, 51867, 68662, 18377, 423, 31250] avg=32768.0000 max=68662 max_rank=4 min=423 min_rank=6 imbalance_ratio=2.095398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[53969, 20849, 43572, 45289, 30906, 54265, 862, 12432] avg=32768.0000 max=54265 max_rank=5 min=862 min_rank=6 imbalance_ratio=1.656036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=28 tokens_per_ep_rank=[6641, 4693, 14916, 81560, 16066, 47999, 28008, 62261] avg=32768.0000 max=81560 max_rank=3 min=4693 min_rank=1 imbalance_ratio=2.489014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31493 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42545 max_load_reduction_pct=0.562460 moved_to_replicas=50182 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=26 tokens_per_ep_rank=[51305, 7457, 32803, 51867, 68662, 18377, 423, 31250] avg=32768.0000 max=68662 max_rank=4 min=423 min_rank=6 imbalance_ratio=2.095398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=27 tokens_per_ep_rank=[29039, 29834, 13486, 21777, 38558, 41062, 70631, 17757] avg=32768.0000 max=70631 max_rank=6 min=13486 min_rank=2 imbalance_ratio=2.155487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=0 min=31165 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=84346 max_load_reduction_pct=0.718193 moved_to_replicas=97475 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] avg=32768.0000 max=33097 max_rank=0 min=31250 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35565 max_load_reduction_pct=0.517972 moved_to_replicas=72546 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[33096, 33095, 33097, 33097, 30906, 33098, 32659, 33096] avg=32768.0000 max=33098 max_rank=5 min=30906 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=21167 max_load_reduction_pct=0.390067 moved_to_replicas=64711 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=48463 max_load_reduction_pct=0.594201 moved_to_replicas=92532 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] avg=32768.0000 max=33097 max_rank=0 min=31250 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35565 max_load_reduction_pct=0.517972 moved_to_replicas=72546 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=26 tokens_per_ep_rank=[51305, 7457, 32803, 51867, 68662, 18377, 423, 31250] avg=32768.0000 max=68662 max_rank=4 min=423 min_rank=6 imbalance_ratio=2.095398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=27 tokens_per_ep_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] avg=32768.0000 max=33097 max_rank=6 min=30850 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37534 max_load_reduction_pct=0.531410 moved_to_replicas=50963 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=25 tokens_per_ep_rank=[6407, 31165, 15561, 12330, 227, 117442, 32787, 46225] avg=32768.0000 max=117442 max_rank=5 min=227 min_rank=4 imbalance_ratio=3.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=26 tokens_per_ep_rank=[51305, 7457, 32803, 51867, 68662, 18377, 423, 31250] avg=32768.0000 max=68662 max_rank=4 min=423 min_rank=6 imbalance_ratio=2.095398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=27 tokens_per_ep_rank=[29039, 29834, 13486, 21777, 38558, 41062, 70631, 17757] avg=32768.0000 max=70631 max_rank=6 min=13486 min_rank=2 imbalance_ratio=2.155487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[31020, 32780, 30327, 27760, 325, 75641, 23558, 40733] avg=32768.0000 max=75641 max_rank=5 min=325 min_rank=4 imbalance_ratio=2.308380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=29 tokens_per_ep_rank=[81, 32850, 22671, 66077, 66694, 26526, 23770, 23475] avg=32768.0000 max=66694 max_rank=4 min=81 min_rank=0 imbalance_ratio=2.035339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] avg=32768.0000 max=33097 max_rank=0 min=31250 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35565 max_load_reduction_pct=0.517972 moved_to_replicas=72546 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=0 min=31165 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=84346 max_load_reduction_pct=0.718193 moved_to_replicas=97475 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=27 tokens_per_ep_rank=[29039, 29834, 13486, 21777, 38558, 41062, 70631, 17757] avg=32768.0000 max=70631 max_rank=6 min=13486 min_rank=2 imbalance_ratio=2.155487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=28 tokens_per_ep_rank=[6641, 4693, 14916, 81560, 16066, 47999, 28008, 62261] avg=32768.0000 max=81560 max_rank=3 min=4693 min_rank=1 imbalance_ratio=2.489014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] avg=32768.0000 max=33097 max_rank=0 min=31250 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35565 max_load_reduction_pct=0.517972 moved_to_replicas=72546 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=27 tokens_per_ep_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] avg=32768.0000 max=33097 max_rank=6 min=30850 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37534 max_load_reduction_pct=0.531410 moved_to_replicas=50963 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[31493, 32780, 33096, 33096, 32391, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31493 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42545 max_load_reduction_pct=0.562460 moved_to_replicas=50182 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=29 tokens_per_ep_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30739 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33597 max_load_reduction_pct=0.503748 moved_to_replicas=66579 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=27 tokens_per_ep_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] avg=32768.0000 max=33097 max_rank=6 min=30850 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37534 max_load_reduction_pct=0.531410 moved_to_replicas=50963 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=48463 max_load_reduction_pct=0.594201 moved_to_replicas=92532 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=27 tokens_per_ep_rank=[29039, 29834, 13486, 21777, 38558, 41062, 70631, 17757] avg=32768.0000 max=70631 max_rank=6 min=13486 min_rank=2 imbalance_ratio=2.155487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=26 tokens_per_ep_rank=[51305, 7457, 32803, 51867, 68662, 18377, 423, 31250] avg=32768.0000 max=68662 max_rank=4 min=423 min_rank=6 imbalance_ratio=2.095398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=27 tokens_per_ep_rank=[29039, 29834, 13486, 21777, 38558, 41062, 70631, 17757] avg=32768.0000 max=70631 max_rank=6 min=13486 min_rank=2 imbalance_ratio=2.155487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=28 tokens_per_ep_rank=[6641, 4693, 14916, 81560, 16066, 47999, 28008, 62261] avg=32768.0000 max=81560 max_rank=3 min=4693 min_rank=1 imbalance_ratio=2.489014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=27 tokens_per_ep_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] avg=32768.0000 max=33097 max_rank=6 min=30850 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37534 max_load_reduction_pct=0.531410 moved_to_replicas=50963 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=30 tokens_per_ep_rank=[1034, 57186, 314, 95352, 43936, 3569, 6681, 54072] avg=32768.0000 max=95352 max_rank=3 min=314 min_rank=2 imbalance_ratio=2.909912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=25 tokens_per_ep_rank=[6407, 31165, 15561, 12330, 227, 117442, 32787, 46225] avg=32768.0000 max=117442 max_rank=5 min=227 min_rank=4 imbalance_ratio=3.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] avg=32768.0000 max=33097 max_rank=0 min=31250 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35565 max_load_reduction_pct=0.517972 moved_to_replicas=72546 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=29 tokens_per_ep_rank=[81, 32850, 22671, 66077, 66694, 26526, 23770, 23475] avg=32768.0000 max=66694 max_rank=4 min=81 min_rank=0 imbalance_ratio=2.035339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=28 tokens_per_ep_rank=[6641, 4693, 14916, 81560, 16066, 47999, 28008, 62261] avg=32768.0000 max=81560 max_rank=3 min=4693 min_rank=1 imbalance_ratio=2.489014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=27 tokens_per_ep_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] avg=32768.0000 max=33097 max_rank=6 min=30850 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37534 max_load_reduction_pct=0.531410 moved_to_replicas=50963 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=48463 max_load_reduction_pct=0.594201 moved_to_replicas=92532 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=30 tokens_per_ep_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30765 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62255 max_load_reduction_pct=0.652897 moved_to_replicas=118162 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=25 tokens_per_ep_rank=[33096, 31165, 33096, 32930, 32878, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=0 min=31165 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=84346 max_load_reduction_pct=0.718193 moved_to_replicas=97475 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=29 tokens_per_ep_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30739 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33597 max_load_reduction_pct=0.503748 moved_to_replicas=66579 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=48463 max_load_reduction_pct=0.594201 moved_to_replicas=92532 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=28 tokens_per_ep_rank=[6641, 4693, 14916, 81560, 16066, 47999, 28008, 62261] avg=32768.0000 max=81560 max_rank=3 min=4693 min_rank=1 imbalance_ratio=2.489014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=27 tokens_per_ep_rank=[29039, 29834, 13486, 21777, 38558, 41062, 70631, 17757] avg=32768.0000 max=70631 max_rank=6 min=13486 min_rank=2 imbalance_ratio=2.155487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=28 tokens_per_ep_rank=[6641, 4693, 14916, 81560, 16066, 47999, 28008, 62261] avg=32768.0000 max=81560 max_rank=3 min=4693 min_rank=1 imbalance_ratio=2.489014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=29 tokens_per_ep_rank=[81, 32850, 22671, 66077, 66694, 26526, 23770, 23475] avg=32768.0000 max=66694 max_rank=4 min=81 min_rank=0 imbalance_ratio=2.035339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=31 tokens_per_ep_rank=[41973, 28862, 81311, 1692, 74617, 219, 32443, 1027] avg=32768.0000 max=81311 max_rank=2 min=219 min_rank=5 imbalance_ratio=2.481415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=48463 max_load_reduction_pct=0.594201 moved_to_replicas=92532 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=26 tokens_per_ep_rank=[51305, 7457, 32803, 51867, 68662, 18377, 423, 31250] avg=32768.0000 max=68662 max_rank=4 min=423 min_rank=6 imbalance_ratio=2.095398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=30 tokens_per_ep_rank=[1034, 57186, 314, 95352, 43936, 3569, 6681, 54072] avg=32768.0000 max=95352 max_rank=3 min=314 min_rank=2 imbalance_ratio=2.909912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=27 tokens_per_ep_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] avg=32768.0000 max=33097 max_rank=6 min=30850 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37534 max_load_reduction_pct=0.531410 moved_to_replicas=50963 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=29 tokens_per_ep_rank=[81, 32850, 22671, 66077, 66694, 26526, 23770, 23475] avg=32768.0000 max=66694 max_rank=4 min=81 min_rank=0 imbalance_ratio=2.035339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=48463 max_load_reduction_pct=0.594201 moved_to_replicas=92532 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=29 tokens_per_ep_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30739 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33597 max_load_reduction_pct=0.503748 moved_to_replicas=66579 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=31 tokens_per_ep_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] avg=32768.0000 max=33097 max_rank=2 min=31914 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48214 max_load_reduction_pct=0.592958 moved_to_replicas=98613 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=26 tokens_per_ep_rank=[33097, 32680, 32803, 33097, 33096, 33096, 33025, 31250] avg=32768.0000 max=33097 max_rank=0 min=31250 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=35565 max_load_reduction_pct=0.517972 moved_to_replicas=72546 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=30 tokens_per_ep_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30765 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62255 max_load_reduction_pct=0.652897 moved_to_replicas=118162 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=29 tokens_per_ep_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30739 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33597 max_load_reduction_pct=0.503748 moved_to_replicas=66579 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=29 tokens_per_ep_rank=[81, 32850, 22671, 66077, 66694, 26526, 23770, 23475] avg=32768.0000 max=66694 max_rank=4 min=81 min_rank=0 imbalance_ratio=2.035339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=28 tokens_per_ep_rank=[6641, 4693, 14916, 81560, 16066, 47999, 28008, 62261] avg=32768.0000 max=81560 max_rank=3 min=4693 min_rank=1 imbalance_ratio=2.489014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=29 tokens_per_ep_rank=[81, 32850, 22671, 66077, 66694, 26526, 23770, 23475] avg=32768.0000 max=66694 max_rank=4 min=81 min_rank=0 imbalance_ratio=2.035339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=30 tokens_per_ep_rank=[1034, 57186, 314, 95352, 43936, 3569, 6681, 54072] avg=32768.0000 max=95352 max_rank=3 min=314 min_rank=2 imbalance_ratio=2.909912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=32 tokens_per_ep_rank=[784, 112, 87820, 40230, 67192, 56153, 791, 9062] avg=32768.0000 max=87820 max_rank=2 min=112 min_rank=1 imbalance_ratio=2.680054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=29 tokens_per_ep_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30739 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33597 max_load_reduction_pct=0.503748 moved_to_replicas=66579 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=31 tokens_per_ep_rank=[41973, 28862, 81311, 1692, 74617, 219, 32443, 1027] avg=32768.0000 max=81311 max_rank=2 min=219 min_rank=5 imbalance_ratio=2.481415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=27 tokens_per_ep_rank=[29039, 29834, 13486, 21777, 38558, 41062, 70631, 17757] avg=32768.0000 max=70631 max_rank=6 min=13486 min_rank=2 imbalance_ratio=2.155487 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=48463 max_load_reduction_pct=0.594201 moved_to_replicas=92532 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=30 tokens_per_ep_rank=[1034, 57186, 314, 95352, 43936, 3569, 6681, 54072] avg=32768.0000 max=95352 max_rank=3 min=314 min_rank=2 imbalance_ratio=2.909912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=29 tokens_per_ep_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30739 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33597 max_load_reduction_pct=0.503748 moved_to_replicas=66579 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=30 tokens_per_ep_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30765 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62255 max_load_reduction_pct=0.652897 moved_to_replicas=118162 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=32 tokens_per_ep_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] avg=32768.0000 max=33096 max_rank=2 min=32018 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54724 max_load_reduction_pct=0.623138 moved_to_replicas=119011 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=31 tokens_per_ep_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] avg=32768.0000 max=33097 max_rank=2 min=31914 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48214 max_load_reduction_pct=0.592958 moved_to_replicas=98613 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=27 tokens_per_ep_rank=[30850, 33096, 33096, 33095, 33096, 33096, 33097, 32718] avg=32768.0000 max=33097 max_rank=6 min=30850 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37534 max_load_reduction_pct=0.531410 moved_to_replicas=50963 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=30 tokens_per_ep_rank=[1034, 57186, 314, 95352, 43936, 3569, 6681, 54072] avg=32768.0000 max=95352 max_rank=3 min=314 min_rank=2 imbalance_ratio=2.909912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=30 tokens_per_ep_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30765 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62255 max_load_reduction_pct=0.652897 moved_to_replicas=118162 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=29 tokens_per_ep_rank=[81, 32850, 22671, 66077, 66694, 26526, 23770, 23475] avg=32768.0000 max=66694 max_rank=4 min=81 min_rank=0 imbalance_ratio=2.035339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=30 tokens_per_ep_rank=[1034, 57186, 314, 95352, 43936, 3569, 6681, 54072] avg=32768.0000 max=95352 max_rank=3 min=314 min_rank=2 imbalance_ratio=2.909912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=31 tokens_per_ep_rank=[41973, 28862, 81311, 1692, 74617, 219, 32443, 1027] avg=32768.0000 max=81311 max_rank=2 min=219 min_rank=5 imbalance_ratio=2.481415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=30 tokens_per_ep_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30765 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62255 max_load_reduction_pct=0.652897 moved_to_replicas=118162 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=33 tokens_per_ep_rank=[21376, 57264, 30072, 32859, 54752, 41643, 23991, 187] avg=32768.0000 max=57264 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.747559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=32 tokens_per_ep_rank=[784, 112, 87820, 40230, 67192, 56153, 791, 9062] avg=32768.0000 max=87820 max_rank=2 min=112 min_rank=1 imbalance_ratio=2.680054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=29 tokens_per_ep_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30739 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33597 max_load_reduction_pct=0.503748 moved_to_replicas=66579 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=28 tokens_per_ep_rank=[6641, 4693, 14916, 81560, 16066, 47999, 28008, 62261] avg=32768.0000 max=81560 max_rank=3 min=4693 min_rank=1 imbalance_ratio=2.489014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=31 tokens_per_ep_rank=[41973, 28862, 81311, 1692, 74617, 219, 32443, 1027] avg=32768.0000 max=81311 max_rank=2 min=219 min_rank=5 imbalance_ratio=2.481415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=30 tokens_per_ep_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30765 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62255 max_load_reduction_pct=0.652897 moved_to_replicas=118162 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=31 tokens_per_ep_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] avg=32768.0000 max=33097 max_rank=2 min=31914 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48214 max_load_reduction_pct=0.592958 moved_to_replicas=98613 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] avg=32768.0000 max=33097 max_rank=1 min=30709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24167 max_load_reduction_pct=0.422028 moved_to_replicas=54371 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=32 tokens_per_ep_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] avg=32768.0000 max=33096 max_rank=2 min=32018 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54724 max_load_reduction_pct=0.623138 moved_to_replicas=119011 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=28 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 33096, 30472, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=48463 max_load_reduction_pct=0.594201 moved_to_replicas=92532 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=31 tokens_per_ep_rank=[41973, 28862, 81311, 1692, 74617, 219, 32443, 1027] avg=32768.0000 max=81311 max_rank=2 min=219 min_rank=5 imbalance_ratio=2.481415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=31 tokens_per_ep_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] avg=32768.0000 max=33097 max_rank=2 min=31914 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48214 max_load_reduction_pct=0.592958 moved_to_replicas=98613 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=30 tokens_per_ep_rank=[1034, 57186, 314, 95352, 43936, 3569, 6681, 54072] avg=32768.0000 max=95352 max_rank=3 min=314 min_rank=2 imbalance_ratio=2.909912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=31 tokens_per_ep_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] avg=32768.0000 max=33097 max_rank=2 min=31914 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48214 max_load_reduction_pct=0.592958 moved_to_replicas=98613 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=31 tokens_per_ep_rank=[41973, 28862, 81311, 1692, 74617, 219, 32443, 1027] avg=32768.0000 max=81311 max_rank=2 min=219 min_rank=5 imbalance_ratio=2.481415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=32 tokens_per_ep_rank=[784, 112, 87820, 40230, 67192, 56153, 791, 9062] avg=32768.0000 max=87820 max_rank=2 min=112 min_rank=1 imbalance_ratio=2.680054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=34 tokens_per_ep_rank=[25907, 33076, 61649, 24632, 36310, 18337, 36494, 25739] avg=32768.0000 max=61649 max_rank=2 min=18337 min_rank=5 imbalance_ratio=1.881378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=33 tokens_per_ep_rank=[21376, 57264, 30072, 32859, 54752, 41643, 23991, 187] avg=32768.0000 max=57264 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.747559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=30 tokens_per_ep_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30765 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62255 max_load_reduction_pct=0.652897 moved_to_replicas=118162 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=29 tokens_per_ep_rank=[81, 32850, 22671, 66077, 66694, 26526, 23770, 23475] avg=32768.0000 max=66694 max_rank=4 min=81 min_rank=0 imbalance_ratio=2.035339 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=32 tokens_per_ep_rank=[784, 112, 87820, 40230, 67192, 56153, 791, 9062] avg=32768.0000 max=87820 max_rank=2 min=112 min_rank=1 imbalance_ratio=2.680054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=31 tokens_per_ep_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] avg=32768.0000 max=33097 max_rank=2 min=31914 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48214 max_load_reduction_pct=0.592958 moved_to_replicas=98613 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=32 tokens_per_ep_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] avg=32768.0000 max=33096 max_rank=2 min=32018 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54724 max_load_reduction_pct=0.623138 moved_to_replicas=119011 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=34 tokens_per_ep_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] avg=32768.0000 max=33096 max_rank=2 min=31069 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28553 max_load_reduction_pct=0.463154 moved_to_replicas=35165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] avg=32768.0000 max=33097 max_rank=1 min=30709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24167 max_load_reduction_pct=0.422028 moved_to_replicas=54371 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=32 tokens_per_ep_rank=[784, 112, 87820, 40230, 67192, 56153, 791, 9062] avg=32768.0000 max=87820 max_rank=2 min=112 min_rank=1 imbalance_ratio=2.680054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=29 tokens_per_ep_rank=[30739, 32850, 33075, 33096, 33095, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30739 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33597 max_load_reduction_pct=0.503748 moved_to_replicas=66579 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=32 tokens_per_ep_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] avg=32768.0000 max=33096 max_rank=2 min=32018 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54724 max_load_reduction_pct=0.623138 moved_to_replicas=119011 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=31 tokens_per_ep_rank=[41973, 28862, 81311, 1692, 74617, 219, 32443, 1027] avg=32768.0000 max=81311 max_rank=2 min=219 min_rank=5 imbalance_ratio=2.481415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=32 tokens_per_ep_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] avg=32768.0000 max=33096 max_rank=2 min=32018 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54724 max_load_reduction_pct=0.623138 moved_to_replicas=119011 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=33 tokens_per_ep_rank=[21376, 57264, 30072, 32859, 54752, 41643, 23991, 187] avg=32768.0000 max=57264 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.747559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=32 tokens_per_ep_rank=[784, 112, 87820, 40230, 67192, 56153, 791, 9062] avg=32768.0000 max=87820 max_rank=2 min=112 min_rank=1 imbalance_ratio=2.680054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=35 tokens_per_ep_rank=[28942, 9208, 32365, 41407, 2892, 15189, 11482, 120659] avg=32768.0000 max=120659 max_rank=7 min=2892 min_rank=4 imbalance_ratio=3.682220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=34 tokens_per_ep_rank=[25907, 33076, 61649, 24632, 36310, 18337, 36494, 25739] avg=32768.0000 max=61649 max_rank=2 min=18337 min_rank=5 imbalance_ratio=1.881378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=30 tokens_per_ep_rank=[1034, 57186, 314, 95352, 43936, 3569, 6681, 54072] avg=32768.0000 max=95352 max_rank=3 min=314 min_rank=2 imbalance_ratio=2.909912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=31 tokens_per_ep_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] avg=32768.0000 max=33097 max_rank=2 min=31914 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48214 max_load_reduction_pct=0.592958 moved_to_replicas=98613 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=33 tokens_per_ep_rank=[21376, 57264, 30072, 32859, 54752, 41643, 23991, 187] avg=32768.0000 max=57264 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.747559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] avg=32768.0000 max=33097 max_rank=1 min=30709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24167 max_load_reduction_pct=0.422028 moved_to_replicas=54371 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=32 tokens_per_ep_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] avg=32768.0000 max=33096 max_rank=2 min=32018 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54724 max_load_reduction_pct=0.623138 moved_to_replicas=119011 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=35 tokens_per_ep_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31203 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=87562 max_load_reduction_pct=0.725698 moved_to_replicas=95874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=34 tokens_per_ep_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] avg=32768.0000 max=33096 max_rank=2 min=31069 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28553 max_load_reduction_pct=0.463154 moved_to_replicas=35165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=33 tokens_per_ep_rank=[21376, 57264, 30072, 32859, 54752, 41643, 23991, 187] avg=32768.0000 max=57264 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.747559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=30 tokens_per_ep_rank=[30765, 33096, 32839, 33096, 33096, 33058, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=30765 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62255 max_load_reduction_pct=0.652897 moved_to_replicas=118162 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] avg=32768.0000 max=33097 max_rank=1 min=30709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24167 max_load_reduction_pct=0.422028 moved_to_replicas=54371 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=32 tokens_per_ep_rank=[784, 112, 87820, 40230, 67192, 56153, 791, 9062] avg=32768.0000 max=87820 max_rank=2 min=112 min_rank=1 imbalance_ratio=2.680054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] avg=32768.0000 max=33097 max_rank=1 min=30709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24167 max_load_reduction_pct=0.422028 moved_to_replicas=54371 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=34 tokens_per_ep_rank=[25907, 33076, 61649, 24632, 36310, 18337, 36494, 25739] avg=32768.0000 max=61649 max_rank=2 min=18337 min_rank=5 imbalance_ratio=1.881378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=33 tokens_per_ep_rank=[21376, 57264, 30072, 32859, 54752, 41643, 23991, 187] avg=32768.0000 max=57264 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.747559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=36 tokens_per_ep_rank=[44511, 49161, 5533, 25142, 32919, 59880, 8023, 36975] avg=32768.0000 max=59880 max_rank=5 min=5533 min_rank=2 imbalance_ratio=1.827393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=35 tokens_per_ep_rank=[28942, 9208, 32365, 41407, 2892, 15189, 11482, 120659] avg=32768.0000 max=120659 max_rank=7 min=2892 min_rank=4 imbalance_ratio=3.682220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=31 tokens_per_ep_rank=[41973, 28862, 81311, 1692, 74617, 219, 32443, 1027] avg=32768.0000 max=81311 max_rank=2 min=219 min_rank=5 imbalance_ratio=2.481415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=32 tokens_per_ep_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] avg=32768.0000 max=33096 max_rank=2 min=32018 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54724 max_load_reduction_pct=0.623138 moved_to_replicas=119011 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=34 tokens_per_ep_rank=[25907, 33076, 61649, 24632, 36310, 18337, 36494, 25739] avg=32768.0000 max=61649 max_rank=2 min=18337 min_rank=5 imbalance_ratio=1.881378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=34 tokens_per_ep_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] avg=32768.0000 max=33096 max_rank=2 min=31069 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28553 max_load_reduction_pct=0.463154 moved_to_replicas=35165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31428 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26783 max_load_reduction_pct=0.447278 moved_to_replicas=58143 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] avg=32768.0000 max=33097 max_rank=1 min=30709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24167 max_load_reduction_pct=0.422028 moved_to_replicas=54371 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=35 tokens_per_ep_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31203 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=87562 max_load_reduction_pct=0.725698 moved_to_replicas=95874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=34 tokens_per_ep_rank=[25907, 33076, 61649, 24632, 36310, 18337, 36494, 25739] avg=32768.0000 max=61649 max_rank=2 min=18337 min_rank=5 imbalance_ratio=1.881378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=31 tokens_per_ep_rank=[33095, 31914, 33097, 33096, 33097, 32307, 32443, 33095] avg=32768.0000 max=33097 max_rank=2 min=31914 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=48214 max_load_reduction_pct=0.592958 moved_to_replicas=98613 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=34 tokens_per_ep_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] avg=32768.0000 max=33096 max_rank=2 min=31069 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28553 max_load_reduction_pct=0.463154 moved_to_replicas=35165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=33 tokens_per_ep_rank=[21376, 57264, 30072, 32859, 54752, 41643, 23991, 187] avg=32768.0000 max=57264 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.747559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=34 tokens_per_ep_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] avg=32768.0000 max=33096 max_rank=2 min=31069 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28553 max_load_reduction_pct=0.463154 moved_to_replicas=35165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=35 tokens_per_ep_rank=[28942, 9208, 32365, 41407, 2892, 15189, 11482, 120659] avg=32768.0000 max=120659 max_rank=7 min=2892 min_rank=4 imbalance_ratio=3.682220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=37 tokens_per_ep_rank=[5397, 27327, 33461, 50772, 33049, 8471, 31613, 72054] avg=32768.0000 max=72054 max_rank=7 min=5397 min_rank=0 imbalance_ratio=2.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=34 tokens_per_ep_rank=[25907, 33076, 61649, 24632, 36310, 18337, 36494, 25739] avg=32768.0000 max=61649 max_rank=2 min=18337 min_rank=5 imbalance_ratio=1.881378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=36 tokens_per_ep_rank=[44511, 49161, 5533, 25142, 32919, 59880, 8023, 36975] avg=32768.0000 max=59880 max_rank=5 min=5533 min_rank=2 imbalance_ratio=1.827393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=32 tokens_per_ep_rank=[784, 112, 87820, 40230, 67192, 56153, 791, 9062] avg=32768.0000 max=87820 max_rank=2 min=112 min_rank=1 imbalance_ratio=2.680054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=35 tokens_per_ep_rank=[28942, 9208, 32365, 41407, 2892, 15189, 11482, 120659] avg=32768.0000 max=120659 max_rank=7 min=2892 min_rank=4 imbalance_ratio=3.682220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] avg=32768.0000 max=33097 max_rank=1 min=30709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24167 max_load_reduction_pct=0.422028 moved_to_replicas=54371 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=37 tokens_per_ep_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] avg=32768.0000 max=33097 max_rank=7 min=31638 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=38957 max_load_reduction_pct=0.540664 moved_to_replicas=56999 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=35 tokens_per_ep_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31203 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=87562 max_load_reduction_pct=0.725698 moved_to_replicas=95874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=34 tokens_per_ep_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] avg=32768.0000 max=33096 max_rank=2 min=31069 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28553 max_load_reduction_pct=0.463154 moved_to_replicas=35165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31428 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26783 max_load_reduction_pct=0.447278 moved_to_replicas=58143 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=35 tokens_per_ep_rank=[28942, 9208, 32365, 41407, 2892, 15189, 11482, 120659] avg=32768.0000 max=120659 max_rank=7 min=2892 min_rank=4 imbalance_ratio=3.682220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=32 tokens_per_ep_rank=[32623, 32024, 33096, 33095, 33096, 33096, 32018, 33096] avg=32768.0000 max=33096 max_rank=2 min=32018 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54724 max_load_reduction_pct=0.623138 moved_to_replicas=119011 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=35 tokens_per_ep_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31203 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=87562 max_load_reduction_pct=0.725698 moved_to_replicas=95874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=35 tokens_per_ep_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31203 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=87562 max_load_reduction_pct=0.725698 moved_to_replicas=95874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=34 tokens_per_ep_rank=[25907, 33076, 61649, 24632, 36310, 18337, 36494, 25739] avg=32768.0000 max=61649 max_rank=2 min=18337 min_rank=5 imbalance_ratio=1.881378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=38 tokens_per_ep_rank=[226, 48089, 13375, 66033, 1036, 30612, 18752, 84021] avg=32768.0000 max=84021 max_rank=7 min=226 min_rank=0 imbalance_ratio=2.564117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=36 tokens_per_ep_rank=[44511, 49161, 5533, 25142, 32919, 59880, 8023, 36975] avg=32768.0000 max=59880 max_rank=5 min=5533 min_rank=2 imbalance_ratio=1.827393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=37 tokens_per_ep_rank=[5397, 27327, 33461, 50772, 33049, 8471, 31613, 72054] avg=32768.0000 max=72054 max_rank=7 min=5397 min_rank=0 imbalance_ratio=2.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=35 tokens_per_ep_rank=[28942, 9208, 32365, 41407, 2892, 15189, 11482, 120659] avg=32768.0000 max=120659 max_rank=7 min=2892 min_rank=4 imbalance_ratio=3.682220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=36 tokens_per_ep_rank=[44511, 49161, 5533, 25142, 32919, 59880, 8023, 36975] avg=32768.0000 max=59880 max_rank=5 min=5533 min_rank=2 imbalance_ratio=1.827393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=33 tokens_per_ep_rank=[21376, 57264, 30072, 32859, 54752, 41643, 23991, 187] avg=32768.0000 max=57264 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.747559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=34 tokens_per_ep_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] avg=32768.0000 max=33096 max_rank=2 min=31069 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28553 max_load_reduction_pct=0.463154 moved_to_replicas=35165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=38 tokens_per_ep_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] avg=32768.0000 max=33097 max_rank=1 min=31667 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50924 max_load_reduction_pct=0.606087 moved_to_replicas=98855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=36 tokens_per_ep_rank=[44511, 49161, 5533, 25142, 32919, 59880, 8023, 36975] avg=32768.0000 max=59880 max_rank=5 min=5533 min_rank=2 imbalance_ratio=1.827393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31428 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26783 max_load_reduction_pct=0.447278 moved_to_replicas=58143 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=37 tokens_per_ep_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] avg=32768.0000 max=33097 max_rank=7 min=31638 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=38957 max_load_reduction_pct=0.540664 moved_to_replicas=56999 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=35 tokens_per_ep_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31203 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=87562 max_load_reduction_pct=0.725698 moved_to_replicas=95874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31428 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26783 max_load_reduction_pct=0.447278 moved_to_replicas=58143 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=33 tokens_per_ep_rank=[33096, 33097, 30709, 32859, 33096, 33097, 33096, 33094] avg=32768.0000 max=33097 max_rank=1 min=30709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=24167 max_load_reduction_pct=0.422028 moved_to_replicas=54371 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31428 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26783 max_load_reduction_pct=0.447278 moved_to_replicas=58143 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=39 tokens_per_ep_rank=[1308, 31258, 37824, 26965, 25699, 64455, 52482, 22153] avg=32768.0000 max=64455 max_rank=5 min=1308 min_rank=0 imbalance_ratio=1.967010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=35 tokens_per_ep_rank=[28942, 9208, 32365, 41407, 2892, 15189, 11482, 120659] avg=32768.0000 max=120659 max_rank=7 min=2892 min_rank=4 imbalance_ratio=3.682220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=37 tokens_per_ep_rank=[5397, 27327, 33461, 50772, 33049, 8471, 31613, 72054] avg=32768.0000 max=72054 max_rank=7 min=5397 min_rank=0 imbalance_ratio=2.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=38 tokens_per_ep_rank=[226, 48089, 13375, 66033, 1036, 30612, 18752, 84021] avg=32768.0000 max=84021 max_rank=7 min=226 min_rank=0 imbalance_ratio=2.564117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=36 tokens_per_ep_rank=[44511, 49161, 5533, 25142, 32919, 59880, 8023, 36975] avg=32768.0000 max=59880 max_rank=5 min=5533 min_rank=2 imbalance_ratio=1.827393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=39 tokens_per_ep_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] avg=32768.0000 max=33098 max_rank=2 min=31258 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=31357 max_load_reduction_pct=0.486494 moved_to_replicas=55473 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=35 tokens_per_ep_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31203 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=87562 max_load_reduction_pct=0.725698 moved_to_replicas=95874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=37 tokens_per_ep_rank=[5397, 27327, 33461, 50772, 33049, 8471, 31613, 72054] avg=32768.0000 max=72054 max_rank=7 min=5397 min_rank=0 imbalance_ratio=2.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=34 tokens_per_ep_rank=[25907, 33076, 61649, 24632, 36310, 18337, 36494, 25739] avg=32768.0000 max=61649 max_rank=2 min=18337 min_rank=5 imbalance_ratio=1.881378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=37 tokens_per_ep_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] avg=32768.0000 max=33097 max_rank=7 min=31638 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=38957 max_load_reduction_pct=0.540664 moved_to_replicas=56999 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=38 tokens_per_ep_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] avg=32768.0000 max=33097 max_rank=1 min=31667 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50924 max_load_reduction_pct=0.606087 moved_to_replicas=98855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=37 tokens_per_ep_rank=[5397, 27327, 33461, 50772, 33049, 8471, 31613, 72054] avg=32768.0000 max=72054 max_rank=7 min=5397 min_rank=0 imbalance_ratio=2.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31428 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26783 max_load_reduction_pct=0.447278 moved_to_replicas=58143 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=37 tokens_per_ep_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] avg=32768.0000 max=33097 max_rank=7 min=31638 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=38957 max_load_reduction_pct=0.540664 moved_to_replicas=56999 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=34 tokens_per_ep_rank=[32519, 33076, 33096, 33096, 33096, 33096, 33096, 31069] avg=32768.0000 max=33096 max_rank=2 min=31069 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28553 max_load_reduction_pct=0.463154 moved_to_replicas=35165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=40 tokens_per_ep_rank=[56806, 575, 20397, 48970, 17760, 63127, 50268, 4241] avg=32768.0000 max=63127 max_rank=5 min=575 min_rank=1 imbalance_ratio=1.926483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=37 tokens_per_ep_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] avg=32768.0000 max=33097 max_rank=7 min=31638 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=38957 max_load_reduction_pct=0.540664 moved_to_replicas=56999 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=36 tokens_per_ep_rank=[44511, 49161, 5533, 25142, 32919, 59880, 8023, 36975] avg=32768.0000 max=59880 max_rank=5 min=5533 min_rank=2 imbalance_ratio=1.827393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=38 tokens_per_ep_rank=[226, 48089, 13375, 66033, 1036, 30612, 18752, 84021] avg=32768.0000 max=84021 max_rank=7 min=226 min_rank=0 imbalance_ratio=2.564117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=39 tokens_per_ep_rank=[1308, 31258, 37824, 26965, 25699, 64455, 52482, 22153] avg=32768.0000 max=64455 max_rank=5 min=1308 min_rank=0 imbalance_ratio=1.967010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=37 tokens_per_ep_rank=[5397, 27327, 33461, 50772, 33049, 8471, 31613, 72054] avg=32768.0000 max=72054 max_rank=7 min=5397 min_rank=0 imbalance_ratio=2.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=40 tokens_per_ep_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] avg=32768.0000 max=33098 max_rank=6 min=30606 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30029 max_load_reduction_pct=0.475692 moved_to_replicas=86787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=38 tokens_per_ep_rank=[226, 48089, 13375, 66033, 1036, 30612, 18752, 84021] avg=32768.0000 max=84021 max_rank=7 min=226 min_rank=0 imbalance_ratio=2.564117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31428 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26783 max_load_reduction_pct=0.447278 moved_to_replicas=58143 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=35 tokens_per_ep_rank=[28942, 9208, 32365, 41407, 2892, 15189, 11482, 120659] avg=32768.0000 max=120659 max_rank=7 min=2892 min_rank=4 imbalance_ratio=3.682220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=38 tokens_per_ep_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] avg=32768.0000 max=33097 max_rank=1 min=31667 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50924 max_load_reduction_pct=0.606087 moved_to_replicas=98855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=39 tokens_per_ep_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] avg=32768.0000 max=33098 max_rank=2 min=31258 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=31357 max_load_reduction_pct=0.486494 moved_to_replicas=55473 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=38 tokens_per_ep_rank=[226, 48089, 13375, 66033, 1036, 30612, 18752, 84021] avg=32768.0000 max=84021 max_rank=7 min=226 min_rank=0 imbalance_ratio=2.564117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=37 tokens_per_ep_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] avg=32768.0000 max=33097 max_rank=7 min=31638 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=38957 max_load_reduction_pct=0.540664 moved_to_replicas=56999 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=38 tokens_per_ep_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] avg=32768.0000 max=33097 max_rank=1 min=31667 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50924 max_load_reduction_pct=0.606087 moved_to_replicas=98855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=41 tokens_per_ep_rank=[20685, 13599, 58384, 7389, 31026, 40780, 7510, 82771] avg=32768.0000 max=82771 max_rank=7 min=7389 min_rank=3 imbalance_ratio=2.525970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=35 tokens_per_ep_rank=[31203, 33096, 32365, 33097, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31203 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=87562 max_load_reduction_pct=0.725698 moved_to_replicas=95874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=38 tokens_per_ep_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] avg=32768.0000 max=33097 max_rank=1 min=31667 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50924 max_load_reduction_pct=0.606087 moved_to_replicas=98855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=37 tokens_per_ep_rank=[5397, 27327, 33461, 50772, 33049, 8471, 31613, 72054] avg=32768.0000 max=72054 max_rank=7 min=5397 min_rank=0 imbalance_ratio=2.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=39 tokens_per_ep_rank=[1308, 31258, 37824, 26965, 25699, 64455, 52482, 22153] avg=32768.0000 max=64455 max_rank=5 min=1308 min_rank=0 imbalance_ratio=1.967010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=40 tokens_per_ep_rank=[56806, 575, 20397, 48970, 17760, 63127, 50268, 4241] avg=32768.0000 max=63127 max_rank=5 min=575 min_rank=1 imbalance_ratio=1.926483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=38 tokens_per_ep_rank=[226, 48089, 13375, 66033, 1036, 30612, 18752, 84021] avg=32768.0000 max=84021 max_rank=7 min=226 min_rank=0 imbalance_ratio=2.564117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=41 tokens_per_ep_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] avg=32768.0000 max=33098 max_rank=1 min=31479 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=49673 max_load_reduction_pct=0.600126 moved_to_replicas=82647 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=39 tokens_per_ep_rank=[1308, 31258, 37824, 26965, 25699, 64455, 52482, 22153] avg=32768.0000 max=64455 max_rank=5 min=1308 min_rank=0 imbalance_ratio=1.967010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=37 tokens_per_ep_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] avg=32768.0000 max=33097 max_rank=7 min=31638 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=38957 max_load_reduction_pct=0.540664 moved_to_replicas=56999 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=39 tokens_per_ep_rank=[1308, 31258, 37824, 26965, 25699, 64455, 52482, 22153] avg=32768.0000 max=64455 max_rank=5 min=1308 min_rank=0 imbalance_ratio=1.967010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=36 tokens_per_ep_rank=[44511, 49161, 5533, 25142, 32919, 59880, 8023, 36975] avg=32768.0000 max=59880 max_rank=5 min=5533 min_rank=2 imbalance_ratio=1.827393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=39 tokens_per_ep_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] avg=32768.0000 max=33098 max_rank=2 min=31258 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=31357 max_load_reduction_pct=0.486494 moved_to_replicas=55473 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=40 tokens_per_ep_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] avg=32768.0000 max=33098 max_rank=6 min=30606 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30029 max_load_reduction_pct=0.475692 moved_to_replicas=86787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=38 tokens_per_ep_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] avg=32768.0000 max=33097 max_rank=1 min=31667 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50924 max_load_reduction_pct=0.606087 moved_to_replicas=98855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=39 tokens_per_ep_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] avg=32768.0000 max=33098 max_rank=2 min=31258 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=31357 max_load_reduction_pct=0.486494 moved_to_replicas=55473 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=42 tokens_per_ep_rank=[17687, 1535, 6658, 116585, 429, 67520, 40153, 11577] avg=32768.0000 max=116585 max_rank=3 min=429 min_rank=4 imbalance_ratio=3.557892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=39 tokens_per_ep_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] avg=32768.0000 max=33098 max_rank=2 min=31258 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=31357 max_load_reduction_pct=0.486494 moved_to_replicas=55473 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=36 tokens_per_ep_rank=[33096, 33095, 32318, 31428, 32919, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31428 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=26783 max_load_reduction_pct=0.447278 moved_to_replicas=58143 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=38 tokens_per_ep_rank=[226, 48089, 13375, 66033, 1036, 30612, 18752, 84021] avg=32768.0000 max=84021 max_rank=7 min=226 min_rank=0 imbalance_ratio=2.564117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=40 tokens_per_ep_rank=[56806, 575, 20397, 48970, 17760, 63127, 50268, 4241] avg=32768.0000 max=63127 max_rank=5 min=575 min_rank=1 imbalance_ratio=1.926483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=41 tokens_per_ep_rank=[20685, 13599, 58384, 7389, 31026, 40780, 7510, 82771] avg=32768.0000 max=82771 max_rank=7 min=7389 min_rank=3 imbalance_ratio=2.525970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=39 tokens_per_ep_rank=[1308, 31258, 37824, 26965, 25699, 64455, 52482, 22153] avg=32768.0000 max=64455 max_rank=5 min=1308 min_rank=0 imbalance_ratio=1.967010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=42 tokens_per_ep_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31929 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83488 max_load_reduction_pct=0.716113 moved_to_replicas=124970 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=40 tokens_per_ep_rank=[56806, 575, 20397, 48970, 17760, 63127, 50268, 4241] avg=32768.0000 max=63127 max_rank=5 min=575 min_rank=1 imbalance_ratio=1.926483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=40 tokens_per_ep_rank=[56806, 575, 20397, 48970, 17760, 63127, 50268, 4241] avg=32768.0000 max=63127 max_rank=5 min=575 min_rank=1 imbalance_ratio=1.926483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=38 tokens_per_ep_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] avg=32768.0000 max=33097 max_rank=1 min=31667 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50924 max_load_reduction_pct=0.606087 moved_to_replicas=98855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=40 tokens_per_ep_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] avg=32768.0000 max=33098 max_rank=6 min=30606 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30029 max_load_reduction_pct=0.475692 moved_to_replicas=86787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=37 tokens_per_ep_rank=[5397, 27327, 33461, 50772, 33049, 8471, 31613, 72054] avg=32768.0000 max=72054 max_rank=7 min=5397 min_rank=0 imbalance_ratio=2.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=41 tokens_per_ep_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] avg=32768.0000 max=33098 max_rank=1 min=31479 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=49673 max_load_reduction_pct=0.600126 moved_to_replicas=82647 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=39 tokens_per_ep_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] avg=32768.0000 max=33098 max_rank=2 min=31258 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=31357 max_load_reduction_pct=0.486494 moved_to_replicas=55473 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=40 tokens_per_ep_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] avg=32768.0000 max=33098 max_rank=6 min=30606 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30029 max_load_reduction_pct=0.475692 moved_to_replicas=86787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=43 tokens_per_ep_rank=[33015, 41282, 7478, 227, 64619, 614, 74363, 40546] avg=32768.0000 max=74363 max_rank=6 min=227 min_rank=3 imbalance_ratio=2.269379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=40 tokens_per_ep_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] avg=32768.0000 max=33098 max_rank=6 min=30606 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30029 max_load_reduction_pct=0.475692 moved_to_replicas=86787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=37 tokens_per_ep_rank=[33095, 31638, 33096, 33096, 33049, 33095, 31978, 33097] avg=32768.0000 max=33097 max_rank=7 min=31638 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=38957 max_load_reduction_pct=0.540664 moved_to_replicas=56999 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=41 tokens_per_ep_rank=[20685, 13599, 58384, 7389, 31026, 40780, 7510, 82771] avg=32768.0000 max=82771 max_rank=7 min=7389 min_rank=3 imbalance_ratio=2.525970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=39 tokens_per_ep_rank=[1308, 31258, 37824, 26965, 25699, 64455, 52482, 22153] avg=32768.0000 max=64455 max_rank=5 min=1308 min_rank=0 imbalance_ratio=1.967010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=43 tokens_per_ep_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31115 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41266 max_load_reduction_pct=0.554927 moved_to_replicas=88426 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=42 tokens_per_ep_rank=[17687, 1535, 6658, 116585, 429, 67520, 40153, 11577] avg=32768.0000 max=116585 max_rank=3 min=429 min_rank=4 imbalance_ratio=3.557892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=40 tokens_per_ep_rank=[56806, 575, 20397, 48970, 17760, 63127, 50268, 4241] avg=32768.0000 max=63127 max_rank=5 min=575 min_rank=1 imbalance_ratio=1.926483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=41 tokens_per_ep_rank=[20685, 13599, 58384, 7389, 31026, 40780, 7510, 82771] avg=32768.0000 max=82771 max_rank=7 min=7389 min_rank=3 imbalance_ratio=2.525970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=41 tokens_per_ep_rank=[20685, 13599, 58384, 7389, 31026, 40780, 7510, 82771] avg=32768.0000 max=82771 max_rank=7 min=7389 min_rank=3 imbalance_ratio=2.525970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=41 tokens_per_ep_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] avg=32768.0000 max=33098 max_rank=1 min=31479 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=49673 max_load_reduction_pct=0.600126 moved_to_replicas=82647 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=39 tokens_per_ep_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] avg=32768.0000 max=33098 max_rank=2 min=31258 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=31357 max_load_reduction_pct=0.486494 moved_to_replicas=55473 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=38 tokens_per_ep_rank=[226, 48089, 13375, 66033, 1036, 30612, 18752, 84021] avg=32768.0000 max=84021 max_rank=7 min=226 min_rank=0 imbalance_ratio=2.564117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=42 tokens_per_ep_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31929 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83488 max_load_reduction_pct=0.716113 moved_to_replicas=124970 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=40 tokens_per_ep_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] avg=32768.0000 max=33098 max_rank=6 min=30606 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30029 max_load_reduction_pct=0.475692 moved_to_replicas=86787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=41 tokens_per_ep_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] avg=32768.0000 max=33098 max_rank=1 min=31479 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=49673 max_load_reduction_pct=0.600126 moved_to_replicas=82647 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=44 tokens_per_ep_rank=[33256, 7765, 31822, 24268, 26071, 58402, 7590, 72970] avg=32768.0000 max=72970 max_rank=7 min=7590 min_rank=6 imbalance_ratio=2.226868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=41 tokens_per_ep_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] avg=32768.0000 max=33098 max_rank=1 min=31479 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=49673 max_load_reduction_pct=0.600126 moved_to_replicas=82647 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=38 tokens_per_ep_rank=[31978, 33097, 33095, 33096, 33095, 33020, 31667, 33096] avg=32768.0000 max=33097 max_rank=1 min=31667 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=50924 max_load_reduction_pct=0.606087 moved_to_replicas=98855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=42 tokens_per_ep_rank=[17687, 1535, 6658, 116585, 429, 67520, 40153, 11577] avg=32768.0000 max=116585 max_rank=3 min=429 min_rank=4 imbalance_ratio=3.557892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=40 tokens_per_ep_rank=[56806, 575, 20397, 48970, 17760, 63127, 50268, 4241] avg=32768.0000 max=63127 max_rank=5 min=575 min_rank=1 imbalance_ratio=1.926483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] avg=32768.0000 max=33097 max_rank=5 min=31746 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39873 max_load_reduction_pct=0.546430 moved_to_replicas=65340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=43 tokens_per_ep_rank=[33015, 41282, 7478, 227, 64619, 614, 74363, 40546] avg=32768.0000 max=74363 max_rank=6 min=227 min_rank=3 imbalance_ratio=2.269379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=41 tokens_per_ep_rank=[20685, 13599, 58384, 7389, 31026, 40780, 7510, 82771] avg=32768.0000 max=82771 max_rank=7 min=7389 min_rank=3 imbalance_ratio=2.525970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=42 tokens_per_ep_rank=[17687, 1535, 6658, 116585, 429, 67520, 40153, 11577] avg=32768.0000 max=116585 max_rank=3 min=429 min_rank=4 imbalance_ratio=3.557892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=42 tokens_per_ep_rank=[17687, 1535, 6658, 116585, 429, 67520, 40153, 11577] avg=32768.0000 max=116585 max_rank=3 min=429 min_rank=4 imbalance_ratio=3.557892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=42 tokens_per_ep_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31929 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83488 max_load_reduction_pct=0.716113 moved_to_replicas=124970 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=40 tokens_per_ep_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] avg=32768.0000 max=33098 max_rank=6 min=30606 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30029 max_load_reduction_pct=0.475692 moved_to_replicas=86787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=39 tokens_per_ep_rank=[1308, 31258, 37824, 26965, 25699, 64455, 52482, 22153] avg=32768.0000 max=64455 max_rank=5 min=1308 min_rank=0 imbalance_ratio=1.967010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=43 tokens_per_ep_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31115 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41266 max_load_reduction_pct=0.554927 moved_to_replicas=88426 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=41 tokens_per_ep_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] avg=32768.0000 max=33098 max_rank=1 min=31479 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=49673 max_load_reduction_pct=0.600126 moved_to_replicas=82647 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=42 tokens_per_ep_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31929 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83488 max_load_reduction_pct=0.716113 moved_to_replicas=124970 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=45 tokens_per_ep_rank=[16788, 58179, 65913, 66058, 9651, 18008, 20885, 6662] avg=32768.0000 max=66058 max_rank=3 min=6662 min_rank=7 imbalance_ratio=2.015930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=42 tokens_per_ep_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31929 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83488 max_load_reduction_pct=0.716113 moved_to_replicas=124970 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=39 tokens_per_ep_rank=[32667, 31258, 33098, 32737, 33096, 33096, 33097, 33095] avg=32768.0000 max=33098 max_rank=2 min=31258 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=31357 max_load_reduction_pct=0.486494 moved_to_replicas=55473 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=43 tokens_per_ep_rank=[33015, 41282, 7478, 227, 64619, 614, 74363, 40546] avg=32768.0000 max=74363 max_rank=6 min=227 min_rank=3 imbalance_ratio=2.269379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=45 tokens_per_ep_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] avg=32768.0000 max=33098 max_rank=4 min=31642 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32960 max_load_reduction_pct=0.498955 moved_to_replicas=90862 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=41 tokens_per_ep_rank=[20685, 13599, 58384, 7389, 31026, 40780, 7510, 82771] avg=32768.0000 max=82771 max_rank=7 min=7389 min_rank=3 imbalance_ratio=2.525970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=44 tokens_per_ep_rank=[33256, 7765, 31822, 24268, 26071, 58402, 7590, 72970] avg=32768.0000 max=72970 max_rank=7 min=7590 min_rank=6 imbalance_ratio=2.226868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=43 tokens_per_ep_rank=[33015, 41282, 7478, 227, 64619, 614, 74363, 40546] avg=32768.0000 max=74363 max_rank=6 min=227 min_rank=3 imbalance_ratio=2.269379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=42 tokens_per_ep_rank=[17687, 1535, 6658, 116585, 429, 67520, 40153, 11577] avg=32768.0000 max=116585 max_rank=3 min=429 min_rank=4 imbalance_ratio=3.557892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=43 tokens_per_ep_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31115 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41266 max_load_reduction_pct=0.554927 moved_to_replicas=88426 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=43 tokens_per_ep_rank=[33015, 41282, 7478, 227, 64619, 614, 74363, 40546] avg=32768.0000 max=74363 max_rank=6 min=227 min_rank=3 imbalance_ratio=2.269379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=40 tokens_per_ep_rank=[56806, 575, 20397, 48970, 17760, 63127, 50268, 4241] avg=32768.0000 max=63127 max_rank=5 min=575 min_rank=1 imbalance_ratio=1.926483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=41 tokens_per_ep_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] avg=32768.0000 max=33098 max_rank=1 min=31479 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=49673 max_load_reduction_pct=0.600126 moved_to_replicas=82647 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] avg=32768.0000 max=33097 max_rank=5 min=31746 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39873 max_load_reduction_pct=0.546430 moved_to_replicas=65340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=43 tokens_per_ep_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31115 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41266 max_load_reduction_pct=0.554927 moved_to_replicas=88426 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=42 tokens_per_ep_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31929 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83488 max_load_reduction_pct=0.716113 moved_to_replicas=124970 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=46 tokens_per_ep_rank=[14201, 16646, 33532, 33134, 29244, 33161, 32606, 69620] avg=32768.0000 max=69620 max_rank=7 min=14201 min_rank=0 imbalance_ratio=2.124634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=43 tokens_per_ep_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31115 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41266 max_load_reduction_pct=0.554927 moved_to_replicas=88426 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=40 tokens_per_ep_rank=[33095, 30606, 33096, 33095, 33094, 33096, 33098, 32964] avg=32768.0000 max=33098 max_rank=6 min=30606 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30029 max_load_reduction_pct=0.475692 moved_to_replicas=86787 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=44 tokens_per_ep_rank=[33256, 7765, 31822, 24268, 26071, 58402, 7590, 72970] avg=32768.0000 max=72970 max_rank=7 min=7590 min_rank=6 imbalance_ratio=2.226868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=46 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] avg=32768.0000 max=33097 max_rank=7 min=30962 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36523 max_load_reduction_pct=0.524605 moved_to_replicas=37063 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=42 tokens_per_ep_rank=[17687, 1535, 6658, 116585, 429, 67520, 40153, 11577] avg=32768.0000 max=116585 max_rank=3 min=429 min_rank=4 imbalance_ratio=3.557892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=45 tokens_per_ep_rank=[16788, 58179, 65913, 66058, 9651, 18008, 20885, 6662] avg=32768.0000 max=66058 max_rank=3 min=6662 min_rank=7 imbalance_ratio=2.015930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=44 tokens_per_ep_rank=[33256, 7765, 31822, 24268, 26071, 58402, 7590, 72970] avg=32768.0000 max=72970 max_rank=7 min=7590 min_rank=6 imbalance_ratio=2.226868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=43 tokens_per_ep_rank=[33015, 41282, 7478, 227, 64619, 614, 74363, 40546] avg=32768.0000 max=74363 max_rank=6 min=227 min_rank=3 imbalance_ratio=2.269379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] avg=32768.0000 max=33097 max_rank=5 min=31746 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39873 max_load_reduction_pct=0.546430 moved_to_replicas=65340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=44 tokens_per_ep_rank=[33256, 7765, 31822, 24268, 26071, 58402, 7590, 72970] avg=32768.0000 max=72970 max_rank=7 min=7590 min_rank=6 imbalance_ratio=2.226868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=41 tokens_per_ep_rank=[20685, 13599, 58384, 7389, 31026, 40780, 7510, 82771] avg=32768.0000 max=82771 max_rank=7 min=7389 min_rank=3 imbalance_ratio=2.525970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=42 tokens_per_ep_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31929 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83488 max_load_reduction_pct=0.716113 moved_to_replicas=124970 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=45 tokens_per_ep_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] avg=32768.0000 max=33098 max_rank=4 min=31642 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32960 max_load_reduction_pct=0.498955 moved_to_replicas=90862 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] avg=32768.0000 max=33097 max_rank=5 min=31746 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39873 max_load_reduction_pct=0.546430 moved_to_replicas=65340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=43 tokens_per_ep_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31115 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41266 max_load_reduction_pct=0.554927 moved_to_replicas=88426 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=47 tokens_per_ep_rank=[146, 28289, 29600, 20293, 26977, 64315, 61561, 30963] avg=32768.0000 max=64315 max_rank=5 min=146 min_rank=0 imbalance_ratio=1.962738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] avg=32768.0000 max=33097 max_rank=5 min=31746 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39873 max_load_reduction_pct=0.546430 moved_to_replicas=65340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=41 tokens_per_ep_rank=[33095, 33098, 33095, 33095, 32090, 33096, 31479, 33096] avg=32768.0000 max=33098 max_rank=1 min=31479 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=49673 max_load_reduction_pct=0.600126 moved_to_replicas=82647 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=45 tokens_per_ep_rank=[16788, 58179, 65913, 66058, 9651, 18008, 20885, 6662] avg=32768.0000 max=66058 max_rank=3 min=6662 min_rank=7 imbalance_ratio=2.015930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=47 tokens_per_ep_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] avg=32768.0000 max=33097 max_rank=1 min=31365 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31218 max_load_reduction_pct=0.485392 moved_to_replicas=59684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=46 tokens_per_ep_rank=[14201, 16646, 33532, 33134, 29244, 33161, 32606, 69620] avg=32768.0000 max=69620 max_rank=7 min=14201 min_rank=0 imbalance_ratio=2.124634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=43 tokens_per_ep_rank=[33015, 41282, 7478, 227, 64619, 614, 74363, 40546] avg=32768.0000 max=74363 max_rank=6 min=227 min_rank=3 imbalance_ratio=2.269379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=45 tokens_per_ep_rank=[16788, 58179, 65913, 66058, 9651, 18008, 20885, 6662] avg=32768.0000 max=66058 max_rank=3 min=6662 min_rank=7 imbalance_ratio=2.015930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=44 tokens_per_ep_rank=[33256, 7765, 31822, 24268, 26071, 58402, 7590, 72970] avg=32768.0000 max=72970 max_rank=7 min=7590 min_rank=6 imbalance_ratio=2.226868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=45 tokens_per_ep_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] avg=32768.0000 max=33098 max_rank=4 min=31642 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32960 max_load_reduction_pct=0.498955 moved_to_replicas=90862 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=45 tokens_per_ep_rank=[16788, 58179, 65913, 66058, 9651, 18008, 20885, 6662] avg=32768.0000 max=66058 max_rank=3 min=6662 min_rank=7 imbalance_ratio=2.015930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=46 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] avg=32768.0000 max=33097 max_rank=7 min=30962 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36523 max_load_reduction_pct=0.524605 moved_to_replicas=37063 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=42 tokens_per_ep_rank=[17687, 1535, 6658, 116585, 429, 67520, 40153, 11577] avg=32768.0000 max=116585 max_rank=3 min=429 min_rank=4 imbalance_ratio=3.557892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=48 tokens_per_ep_rank=[16589, 32725, 8111, 18980, 50827, 45995, 50096, 38821] avg=32768.0000 max=50827 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.551117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=43 tokens_per_ep_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31115 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41266 max_load_reduction_pct=0.554927 moved_to_replicas=88426 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=45 tokens_per_ep_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] avg=32768.0000 max=33098 max_rank=4 min=31642 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32960 max_load_reduction_pct=0.498955 moved_to_replicas=90862 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] avg=32768.0000 max=33097 max_rank=5 min=31746 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39873 max_load_reduction_pct=0.546430 moved_to_replicas=65340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=45 tokens_per_ep_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] avg=32768.0000 max=33098 max_rank=4 min=31642 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32960 max_load_reduction_pct=0.498955 moved_to_replicas=90862 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=42 tokens_per_ep_rank=[33096, 31929, 32745, 33095, 31991, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31929 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=83488 max_load_reduction_pct=0.716113 moved_to_replicas=124970 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=46 tokens_per_ep_rank=[14201, 16646, 33532, 33134, 29244, 33161, 32606, 69620] avg=32768.0000 max=69620 max_rank=7 min=14201 min_rank=0 imbalance_ratio=2.124634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=48 tokens_per_ep_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31567 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17730 max_load_reduction_pct=0.348830 moved_to_replicas=53355 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=47 tokens_per_ep_rank=[146, 28289, 29600, 20293, 26977, 64315, 61561, 30963] avg=32768.0000 max=64315 max_rank=5 min=146 min_rank=0 imbalance_ratio=1.962738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=44 tokens_per_ep_rank=[33256, 7765, 31822, 24268, 26071, 58402, 7590, 72970] avg=32768.0000 max=72970 max_rank=7 min=7590 min_rank=6 imbalance_ratio=2.226868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=45 tokens_per_ep_rank=[16788, 58179, 65913, 66058, 9651, 18008, 20885, 6662] avg=32768.0000 max=66058 max_rank=3 min=6662 min_rank=7 imbalance_ratio=2.015930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=46 tokens_per_ep_rank=[14201, 16646, 33532, 33134, 29244, 33161, 32606, 69620] avg=32768.0000 max=69620 max_rank=7 min=14201 min_rank=0 imbalance_ratio=2.124634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=46 tokens_per_ep_rank=[14201, 16646, 33532, 33134, 29244, 33161, 32606, 69620] avg=32768.0000 max=69620 max_rank=7 min=14201 min_rank=0 imbalance_ratio=2.124634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=46 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] avg=32768.0000 max=33097 max_rank=7 min=30962 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36523 max_load_reduction_pct=0.524605 moved_to_replicas=37063 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=47 tokens_per_ep_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] avg=32768.0000 max=33097 max_rank=1 min=31365 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31218 max_load_reduction_pct=0.485392 moved_to_replicas=59684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=43 tokens_per_ep_rank=[33015, 41282, 7478, 227, 64619, 614, 74363, 40546] avg=32768.0000 max=74363 max_rank=6 min=227 min_rank=3 imbalance_ratio=2.269379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] avg=32768.0000 max=33097 max_rank=5 min=31746 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39873 max_load_reduction_pct=0.546430 moved_to_replicas=65340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=45 tokens_per_ep_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] avg=32768.0000 max=33098 max_rank=4 min=31642 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32960 max_load_reduction_pct=0.498955 moved_to_replicas=90862 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=46 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] avg=32768.0000 max=33097 max_rank=7 min=30962 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36523 max_load_reduction_pct=0.524605 moved_to_replicas=37063 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=46 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] avg=32768.0000 max=33097 max_rank=7 min=30962 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36523 max_load_reduction_pct=0.524605 moved_to_replicas=37063 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=43 tokens_per_ep_rank=[33015, 33096, 33097, 32533, 33096, 31115, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=31115 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41266 max_load_reduction_pct=0.554927 moved_to_replicas=88426 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=47 tokens_per_ep_rank=[146, 28289, 29600, 20293, 26977, 64315, 61561, 30963] avg=32768.0000 max=64315 max_rank=5 min=146 min_rank=0 imbalance_ratio=1.962738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=48 tokens_per_ep_rank=[16589, 32725, 8111, 18980, 50827, 45995, 50096, 38821] avg=32768.0000 max=50827 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.551117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=46 tokens_per_ep_rank=[14201, 16646, 33532, 33134, 29244, 33161, 32606, 69620] avg=32768.0000 max=69620 max_rank=7 min=14201 min_rank=0 imbalance_ratio=2.124634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=47 tokens_per_ep_rank=[146, 28289, 29600, 20293, 26977, 64315, 61561, 30963] avg=32768.0000 max=64315 max_rank=5 min=146 min_rank=0 imbalance_ratio=1.962738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=47 tokens_per_ep_rank=[146, 28289, 29600, 20293, 26977, 64315, 61561, 30963] avg=32768.0000 max=64315 max_rank=5 min=146 min_rank=0 imbalance_ratio=1.962738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=45 tokens_per_ep_rank=[16788, 58179, 65913, 66058, 9651, 18008, 20885, 6662] avg=32768.0000 max=66058 max_rank=3 min=6662 min_rank=7 imbalance_ratio=2.015930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=47 tokens_per_ep_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] avg=32768.0000 max=33097 max_rank=1 min=31365 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31218 max_load_reduction_pct=0.485392 moved_to_replicas=59684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=48 tokens_per_ep_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31567 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17730 max_load_reduction_pct=0.348830 moved_to_replicas=53355 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=46 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] avg=32768.0000 max=33097 max_rank=7 min=30962 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36523 max_load_reduction_pct=0.524605 moved_to_replicas=37063 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=44 tokens_per_ep_rank=[33256, 7765, 31822, 24268, 26071, 58402, 7590, 72970] avg=32768.0000 max=72970 max_rank=7 min=7590 min_rank=6 imbalance_ratio=2.226868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=47 tokens_per_ep_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] avg=32768.0000 max=33097 max_rank=1 min=31365 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31218 max_load_reduction_pct=0.485392 moved_to_replicas=59684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=47 tokens_per_ep_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] avg=32768.0000 max=33097 max_rank=1 min=31365 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31218 max_load_reduction_pct=0.485392 moved_to_replicas=59684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=45 tokens_per_ep_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] avg=32768.0000 max=33098 max_rank=4 min=31642 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32960 max_load_reduction_pct=0.498955 moved_to_replicas=90862 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=48 tokens_per_ep_rank=[16589, 32725, 8111, 18980, 50827, 45995, 50096, 38821] avg=32768.0000 max=50827 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.551117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=44 tokens_per_ep_rank=[33096, 33096, 31822, 33095, 31746, 33097, 33095, 33097] avg=32768.0000 max=33097 max_rank=5 min=31746 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39873 max_load_reduction_pct=0.546430 moved_to_replicas=65340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=47 tokens_per_ep_rank=[146, 28289, 29600, 20293, 26977, 64315, 61561, 30963] avg=32768.0000 max=64315 max_rank=5 min=146 min_rank=0 imbalance_ratio=1.962738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=48 tokens_per_ep_rank=[16589, 32725, 8111, 18980, 50827, 45995, 50096, 38821] avg=32768.0000 max=50827 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.551117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=48 tokens_per_ep_rank=[16589, 32725, 8111, 18980, 50827, 45995, 50096, 38821] avg=32768.0000 max=50827 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.551117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=48 tokens_per_ep_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31567 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17730 max_load_reduction_pct=0.348830 moved_to_replicas=53355 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=46 tokens_per_ep_rank=[14201, 16646, 33532, 33134, 29244, 33161, 32606, 69620] avg=32768.0000 max=69620 max_rank=7 min=14201 min_rank=0 imbalance_ratio=2.124634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=47 tokens_per_ep_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] avg=32768.0000 max=33097 max_rank=1 min=31365 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31218 max_load_reduction_pct=0.485392 moved_to_replicas=59684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=48 tokens_per_ep_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31567 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17730 max_load_reduction_pct=0.348830 moved_to_replicas=53355 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=48 tokens_per_ep_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31567 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17730 max_load_reduction_pct=0.348830 moved_to_replicas=53355 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=45 tokens_per_ep_rank=[16788, 58179, 65913, 66058, 9651, 18008, 20885, 6662] avg=32768.0000 max=66058 max_rank=3 min=6662 min_rank=7 imbalance_ratio=2.015930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=46 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] avg=32768.0000 max=33097 max_rank=7 min=30962 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36523 max_load_reduction_pct=0.524605 moved_to_replicas=37063 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=45 tokens_per_ep_rank=[31945, 33096, 33096, 33095, 33098, 33095, 31642, 33077] avg=32768.0000 max=33098 max_rank=4 min=31642 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32960 max_load_reduction_pct=0.498955 moved_to_replicas=90862 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=48 tokens_per_ep_rank=[16589, 32725, 8111, 18980, 50827, 45995, 50096, 38821] avg=32768.0000 max=50827 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.551117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=47 tokens_per_ep_rank=[146, 28289, 29600, 20293, 26977, 64315, 61561, 30963] avg=32768.0000 max=64315 max_rank=5 min=146 min_rank=0 imbalance_ratio=1.962738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=48 tokens_per_ep_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31567 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17730 max_load_reduction_pct=0.348830 moved_to_replicas=53355 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=46 tokens_per_ep_rank=[14201, 16646, 33532, 33134, 29244, 33161, 32606, 69620] avg=32768.0000 max=69620 max_rank=7 min=14201 min_rank=0 imbalance_ratio=2.124634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=47 tokens_per_ep_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] avg=32768.0000 max=33097 max_rank=1 min=31365 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31218 max_load_reduction_pct=0.485392 moved_to_replicas=59684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=46 tokens_per_ep_rank=[33096, 33095, 33096, 33096, 30962, 33096, 32606, 33097] avg=32768.0000 max=33097 max_rank=7 min=30962 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=36523 max_load_reduction_pct=0.524605 moved_to_replicas=37063 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=48 tokens_per_ep_rank=[16589, 32725, 8111, 18980, 50827, 45995, 50096, 38821] avg=32768.0000 max=50827 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.551117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=47 tokens_per_ep_rank=[146, 28289, 29600, 20293, 26977, 64315, 61561, 30963] avg=32768.0000 max=64315 max_rank=5 min=146 min_rank=0 imbalance_ratio=1.962738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=48 tokens_per_ep_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31567 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17730 max_load_reduction_pct=0.348830 moved_to_replicas=53355 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=47 tokens_per_ep_rank=[31365, 33097, 33096, 33096, 33095, 33096, 33096, 32203] avg=32768.0000 max=33097 max_rank=1 min=31365 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31218 max_load_reduction_pct=0.485392 moved_to_replicas=59684 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=48 tokens_per_ep_rank=[16589, 32725, 8111, 18980, 50827, 45995, 50096, 38821] avg=32768.0000 max=50827 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.551117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=48 tokens_per_ep_rank=[33096, 32725, 31567, 32371, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31567 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=17730 max_load_reduction_pct=0.348830 moved_to_replicas=53355 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=48 total_expected_recv=1572583 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=355072 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=48 total_expected_recv=1565470 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=360832 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=48 total_expected_recv=1581676 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344832 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=48 total_expected_recv=1570165 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=357376 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=48 total_expected_recv=1568512 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=356480 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=48 total_expected_recv=1573310 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=353152 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=48 total_expected_recv=1577019 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=349952 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=48 total_expected_recv=1574177 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=354048 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:47:00] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 998.3 | throughput per GPU (TFLOP/s/GPU): 89.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008715E+01 | loss scale: 1.0 | grad norm: 180.510 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=31011 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=31008 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=31008 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=31486 expected_overflow=0 exact_recv_per_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=31486 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=31257 expected_overflow=0 exact_recv_per_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=31257 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32274 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=31759 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32274 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=31759 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32405 expected_overflow=0 exact_recv_per_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=31615 expected_overflow=0 exact_recv_per_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32405 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=31615 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32259 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=31917 expected_overflow=0 exact_recv_per_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32259 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=31978 expected_overflow=0 exact_recv_per_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=31978 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32207 expected_overflow=0 exact_recv_per_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=31917 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32207 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32000 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32000 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=31871 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=31839 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=31871 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=31839 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33092 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33092 actual_valid_recv=34944 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34944 slack_slots=6016 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=31968 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=31968 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=30764 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=30764 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=30769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=30769 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=35072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=35072 slack_slots=5888 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=31144 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=31144 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=31668 expected_overflow=0 exact_recv_per_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=31668 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=31643 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=31643 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32212 expected_overflow=0 exact_recv_per_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32212 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=31886 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=31886 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32202 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32202 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=31882 expected_overflow=0 exact_recv_per_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=31882 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32012 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32012 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32521 expected_overflow=0 exact_recv_per_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32521 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=40960 expected_recv=31595 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=40960 expected_recv=31586 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=40960 expected_recv=31586 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32007 expected_overflow=0 exact_recv_per_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32007 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=40960 expected_recv=31595 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=40960 expected_recv=31399 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=40960 expected_recv=31399 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=40960 expected_recv=31391 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=40960 expected_recv=31391 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=40960 expected_recv=31604 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=40960 expected_recv=31604 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33059 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33059 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=40960 expected_recv=31621 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=40960 expected_recv=31621 actual_valid_recv=32256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32336 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32336 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32324 expected_overflow=0 exact_recv_per_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32324 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32127 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32557 expected_overflow=0 exact_recv_per_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32557 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32127 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32377 expected_overflow=0 exact_recv_per_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32377 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33068 expected_overflow=0 exact_recv_per_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32106 expected_overflow=0 exact_recv_per_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33068 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32355 expected_overflow=0 exact_recv_per_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32355 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32106 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32174 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32484 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32484 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=40960 expected_recv=31032 expected_overflow=0 exact_recv_per_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=40960 expected_recv=31032 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32483 expected_overflow=0 exact_recv_per_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32483 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=40960 expected_recv=31022 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=40960 expected_recv=31022 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=40960 expected_recv=31566 expected_overflow=0 exact_recv_per_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32936 expected_overflow=0 exact_recv_per_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=40960 expected_recv=31566 actual_valid_recv=31872 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=40960 expected_recv=31584 expected_overflow=0 exact_recv_per_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=40960 expected_recv=31584 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32936 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=40960 expected_recv=31819 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=40960 expected_recv=31819 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=40960 expected_recv=31816 expected_overflow=0 exact_recv_per_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=40960 expected_recv=31816 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32264 expected_overflow=0 exact_recv_per_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32264 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32404 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32343 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32404 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32343 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32411 expected_overflow=0 exact_recv_per_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32411 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=24 calls=1 budget=40960 expected_recv=31117 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=24 calls=1 budget=40960 expected_recv=31117 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=24 calls=1 budget=40960 expected_recv=31116 expected_overflow=0 exact_recv_per_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=24 calls=1 budget=40960 expected_recv=31116 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33020 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33039 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32122 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33020 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32122 expected_overflow=0 exact_recv_per_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32122 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32374 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32374 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=26 calls=1 budget=40960 expected_recv=31868 expected_overflow=0 exact_recv_per_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=26 calls=1 budget=40960 expected_recv=31868 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32959 expected_overflow=0 exact_recv_per_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32959 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33036 expected_overflow=0 exact_recv_per_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33036 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=25 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=26 calls=1 budget=40960 expected_recv=31859 expected_overflow=0 exact_recv_per_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=26 calls=1 budget=40960 expected_recv=31859 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32399 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32399 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=26 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32497 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32497 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=28 calls=1 budget=40960 expected_recv=31947 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=28 calls=1 budget=40960 expected_recv=31947 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32008 expected_overflow=0 exact_recv_per_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32008 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32255 expected_overflow=0 exact_recv_per_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=30 calls=1 budget=40960 expected_recv=31935 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=30 calls=1 budget=40960 expected_recv=31935 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32261 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32255 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32548 expected_overflow=0 exact_recv_per_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32593 expected_overflow=0 exact_recv_per_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32548 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32593 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32261 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33088 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32948 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33088 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32948 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=30 calls=1 budget=40960 expected_recv=31944 expected_overflow=0 exact_recv_per_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=30 calls=1 budget=40960 expected_recv=31944 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=31 calls=1 budget=40960 expected_recv=31901 expected_overflow=0 exact_recv_per_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=31 calls=1 budget=40960 expected_recv=31901 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=31 calls=1 budget=40960 expected_recv=31892 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=31 calls=1 budget=40960 expected_recv=31892 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32408 expected_overflow=0 exact_recv_per_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32408 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32155 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32155 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=33 calls=1 budget=40960 expected_recv=31863 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=33 calls=1 budget=40960 expected_recv=31863 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32239 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32162 expected_overflow=0 exact_recv_per_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32162 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32239 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32280 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32280 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32246 expected_overflow=0 exact_recv_per_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32246 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=33 calls=1 budget=40960 expected_recv=31859 expected_overflow=0 exact_recv_per_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=33 calls=1 budget=40960 expected_recv=31859 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32104 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32104 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32454 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32454 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32104 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32104 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32051 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32051 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32288 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32288 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32245 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32245 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=39 calls=1 budget=40960 expected_recv=31627 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=39 calls=1 budget=40960 expected_recv=31627 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=39 calls=1 budget=40960 expected_recv=31644 expected_overflow=0 exact_recv_per_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=39 calls=1 budget=40960 expected_recv=31644 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32468 expected_overflow=0 exact_recv_per_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32468 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32423 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32423 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32154 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32052 expected_overflow=0 exact_recv_per_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32245 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32052 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32245 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32154 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=42 calls=1 budget=40960 expected_recv=31397 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=42 calls=1 budget=40960 expected_recv=31397 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=43 calls=1 budget=40960 expected_recv=31237 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=43 calls=1 budget=40960 expected_recv=31237 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32412 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=42 calls=1 budget=40960 expected_recv=31407 expected_overflow=0 exact_recv_per_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=42 calls=1 budget=40960 expected_recv=31407 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32084 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32520 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=43 calls=1 budget=40960 expected_recv=31228 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32520 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=45 calls=1 budget=40960 expected_recv=31973 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=43 calls=1 budget=40960 expected_recv=31228 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33029 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=45 calls=1 budget=40960 expected_recv=31982 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=46 calls=1 budget=40960 expected_recv=31490 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33029 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=45 calls=1 budget=40960 expected_recv=31982 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=46 calls=1 budget=40960 expected_recv=31490 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=45 calls=1 budget=40960 expected_recv=31973 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=46 calls=1 budget=40960 expected_recv=31463 expected_overflow=0 exact_recv_per_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=46 calls=1 budget=40960 expected_recv=31463 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=47 calls=1 budget=40960 expected_recv=31701 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=47 calls=1 budget=40960 expected_recv=31701 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32521 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32521 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32400 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[39385, 21239, 32560, 21759, 45306, 25241, 43122, 33532] avg=32768.0000 max=45306 max_rank=4 min=21239 min_rank=1 imbalance_ratio=1.382629 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32400 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[39385, 21239, 32560, 21759, 45306, 25241, 43122, 33532] avg=32768.0000 max=45306 max_rank=4 min=21239 min_rank=1 imbalance_ratio=1.382629 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31008 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=12208 max_load_reduction_pct=0.269457 moved_to_replicas=28961 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31008 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=12208 max_load_reduction_pct=0.269457 moved_to_replicas=28961 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[39385, 21239, 32560, 21759, 45306, 25241, 43122, 33532] avg=32768.0000 max=45306 max_rank=4 min=21239 min_rank=1 imbalance_ratio=1.382629 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[39385, 21239, 32560, 21759, 45306, 25241, 43122, 33532] avg=32768.0000 max=45306 max_rank=4 min=21239 min_rank=1 imbalance_ratio=1.382629 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33060 expected_overflow=0 exact_recv_per_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[39417, 21161, 32557, 21810, 45308, 25263, 43195, 33433] avg=32768.0000 max=45308 max_rank=4 min=21161 min_rank=1 imbalance_ratio=1.382690 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 32557, 31011, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31011 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=12212 max_load_reduction_pct=0.269533 moved_to_replicas=28969 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[8494, 32855, 35136, 178, 18830, 94655, 22493, 49503] avg=32768.0000 max=94655 max_rank=5 min=178 min_rank=3 imbalance_ratio=2.888641 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32855, 33096, 32551, 31257, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31257 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61558 max_load_reduction_pct=0.650341 moved_to_replicas=80006 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[17297, 38291, 32274, 63075, 66326, 159, 1266, 43456] avg=32768.0000 max=66326 max_rank=4 min=159 min_rank=5 imbalance_ratio=2.024109 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32850, 33097, 32274, 33096, 33096, 32875, 31759, 33097] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33229 max_load_reduction_pct=0.500995 moved_to_replicas=78764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[43, 32259, 45286, 70336, 61, 32815, 42803, 38541] avg=32768.0000 max=70336 max_rank=3 min=43 min_rank=0 imbalance_ratio=2.146484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32768, 32259, 33097, 33096, 31917, 32815, 33095, 33097] avg=32768.0000 max=33097 max_rank=2 min=31917 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37239 max_load_reduction_pct=0.529444 moved_to_replicas=64582 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[65704, 2063, 73542, 55407, 31839, 123, 2290, 31176] avg=32768.0000 max=73542 max_rank=2 min=123 min_rank=5 imbalance_ratio=2.244324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33094, 33097, 33097, 31839, 32825, 33096, 32000] avg=32768.0000 max=33097 max_rank=2 min=31839 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40445 max_load_reduction_pct=0.549958 moved_to_replicas=95365 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[3085, 33652, 164, 65104, 30213, 100, 41197, 88629] avg=32768.0000 max=88629 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.704742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33092, 33096, 30769, 32803, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30769 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55533 max_load_reduction_pct=0.626578 moved_to_replicas=96198 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[42772, 32771, 253, 33236, 50, 50674, 23449, 78939] avg=32768.0000 max=78939 max_rank=7 min=50 min_rank=4 imbalance_ratio=2.409027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32771, 31144, 33096, 32749, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=0 min=31144 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45842 max_load_reduction_pct=0.580727 moved_to_replicas=73237 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[29867, 261, 25510, 61859, 24, 65548, 31643, 47432] avg=32768.0000 max=65548 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000366 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32545, 33095, 33096, 33096, 32476, 33096, 31643, 33097] avg=32768.0000 max=33097 max_rank=7 min=31643 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32451 max_load_reduction_pct=0.495072 moved_to_replicas=75551 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[64993, 315, 2365, 63315, 32790, 32765, 32856, 32745] avg=32768.0000 max=64993 max_rank=0 min=315 min_rank=1 imbalance_ratio=1.983429 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32212, 32584, 33096, 32790, 32765, 32856, 32745] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31897 max_load_reduction_pct=0.490776 moved_to_replicas=62116 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 4874, 89243, 256, 65502, 4232] avg=32768.0000 max=89243 max_rank=4 min=256 min_rank=5 imbalance_ratio=2.723480 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32684, 32675, 32678, 33098, 33094, 32933, 33096, 31886] avg=32768.0000 max=33098 max_rank=3 min=31886 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=56145 max_load_reduction_pct=0.629125 moved_to_replicas=88553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[205, 1355, 44768, 32012, 52894, 301, 98007, 32602] avg=32768.0000 max=98007 max_rank=6 min=205 min_rank=0 imbalance_ratio=2.990936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32895, 32826, 33096, 32012, 33095, 32522, 33096, 32602] avg=32768.0000 max=33096 max_rank=2 min=32012 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96381 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[168, 76966, 93752, 58405, 27, 32742, 84, 0] avg=32768.0000 max=93752 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.861084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[33097, 33096, 33095, 33095, 31595, 32742, 32761, 32663] avg=32768.0000 max=33097 max_rank=0 min=31595 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60655 max_load_reduction_pct=0.646973 moved_to_replicas=129835 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[3203, 33421, 140, 65347, 42572, 98019, 202, 19240] avg=32768.0000 max=98019 max_rank=5 min=140 min_rank=2 imbalance_ratio=2.991302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32798, 33096, 33096, 33096, 32467, 31399] avg=32768.0000 max=33096 max_rank=0 min=31399 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106975 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32808, 65332, 97019, 3810, 90, 118, 33059, 29908] avg=32768.0000 max=97019 max_rank=2 min=90 min_rank=4 imbalance_ratio=2.960785 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32808, 33096, 33096, 33096, 32527, 31604, 33059, 32858] avg=32768.0000 max=33096 max_rank=1 min=31604 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63923 max_load_reduction_pct=0.658871 moved_to_replicas=96159 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[31894, 65449, 65187, 1779, 32749, 92, 32658, 32336] avg=32768.0000 max=65449 max_rank=1 min=92 min_rank=5 imbalance_ratio=1.997345 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32668, 33096, 33096, 33096, 32749, 32445, 32658, 32336] avg=32768.0000 max=33096 max_rank=1 min=32336 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=64444 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[65186, 32777, 32822, 32660, 65725, 471, 32496, 7] avg=32768.0000 max=65725 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32777, 32822, 32660, 33096, 32561, 32496, 32636] avg=32768.0000 max=33096 max_rank=0 min=32496 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32629 max_load_reduction_pct=0.496447 moved_to_replicas=64719 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[32788, 21474, 40373, 128, 32674, 65095, 32913, 36699] avg=32768.0000 max=65095 max_rank=5 min=128 min_rank=3 imbalance_ratio=1.986542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[32788, 32355, 33095, 32127, 32674, 33096, 32913, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31999 max_load_reduction_pct=0.491574 moved_to_replicas=42879 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[132, 79, 134, 65464, 65136, 65400, 33063, 32736] avg=32768.0000 max=65464 max_rank=3 min=79 min_rank=1 imbalance_ratio=1.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[32436, 32447, 32174, 33096, 33096, 33096, 33063, 32736] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32368 max_load_reduction_pct=0.494440 moved_to_replicas=96712 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[1712, 32576, 117, 65346, 176, 65463, 32682, 64072] avg=32768.0000 max=65463 max_rank=5 min=117 min_rank=2 imbalance_ratio=1.997772 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[32688, 32576, 32484, 33096, 32426, 33096, 32682, 33096] avg=32768.0000 max=33096 max_rank=3 min=32426 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=95593 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[35445, 269, 93062, 1810, 341, 624, 65405, 65188] avg=32768.0000 max=93062 max_rank=2 min=269 min_rank=1 imbalance_ratio=2.840027 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32709, 33096, 33096, 31022, 32933, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31022 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59966 max_load_reduction_pct=0.644366 moved_to_replicas=126716 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[32756, 57, 15, 78061, 207, 85455, 313, 65280] avg=32768.0000 max=85455 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.607880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[32756, 31584, 32585, 33096, 32836, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31584 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=52359 max_load_reduction_pct=0.612708 moved_to_replicas=129508 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[32918, 97149, 205, 1626, 716, 65299, 63890, 341] avg=32768.0000 max=97149 max_rank=1 min=205 min_rank=2 imbalance_ratio=2.964752 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[32918, 33096, 32780, 32420, 32919, 33096, 33096, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=127050 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[108, 32765, 65392, 32913, 32288, 36066, 29461, 33151] avg=32768.0000 max=65392 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.995605 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[32404, 32765, 33096, 32913, 32343, 33096, 32431, 33096] avg=32768.0000 max=33096 max_rank=2 min=32343 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=35321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32311, 32861, 65289, 32422, 1325, 32753] avg=32768.0000 max=65289 max_rank=4 min=1325 min_rank=6 imbalance_ratio=1.992462 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[32533, 32650, 32733, 32861, 33096, 32422, 33096, 32753] avg=32768.0000 max=33096 max_rank=4 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32193 max_load_reduction_pct=0.493085 moved_to_replicas=32193 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=25 tokens_per_ep_rank=[10129, 32763, 39467, 65297, 1063, 65617, 15025, 32783] avg=32768.0000 max=65617 max_rank=5 min=1063 min_rank=4 imbalance_ratio=2.002472 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=25 tokens_per_ep_rank=[33095, 32763, 33098, 33096, 33096, 33096, 31117, 32783] avg=32768.0000 max=33098 max_rank=2 min=31117 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32519 max_load_reduction_pct=0.495588 moved_to_replicas=71093 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=26 tokens_per_ep_rank=[32122, 231, 33001, 32639, 97905, 461, 32765, 33020] avg=32768.0000 max=97905 max_rank=4 min=231 min_rank=1 imbalance_ratio=2.987823 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32801, 33001, 32639, 33096, 32700, 32765, 33020] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64809 max_load_reduction_pct=0.661958 moved_to_replicas=64809 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 44758, 268, 65202, 33039, 53475, 578] avg=32768.0000 max=65202 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=27 tokens_per_ep_rank=[32956, 31868, 33097, 32374, 33096, 33039, 33096, 32618] avg=32768.0000 max=33097 max_rank=2 min=31868 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32105 max_load_reduction_pct=0.492393 moved_to_replicas=64147 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=28 tokens_per_ep_rank=[664, 32637, 735, 32587, 32753, 65207, 32399, 65162] avg=32768.0000 max=65207 max_rank=5 min=664 min_rank=0 imbalance_ratio=1.989960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=28 tokens_per_ep_rank=[32775, 32637, 32801, 32587, 32753, 33096, 32399, 33096] avg=32768.0000 max=33096 max_rank=5 min=32399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32111 max_load_reduction_pct=0.492447 moved_to_replicas=64177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=29 tokens_per_ep_rank=[31906, 32925, 64872, 50505, 64737, 15728, 750, 721] avg=32768.0000 max=64872 max_rank=2 min=721 min_rank=7 imbalance_ratio=1.979736 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=29 tokens_per_ep_rank=[31947, 32925, 33096, 33096, 33096, 33096, 32391, 32497] avg=32768.0000 max=33096 max_rank=2 min=31947 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31776 max_load_reduction_pct=0.489826 moved_to_replicas=80826 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=30 tokens_per_ep_rank=[29452, 32600, 32611, 65245, 65189, 32729, 3812, 506] avg=32768.0000 max=65245 max_rank=3 min=506 min_rank=7 imbalance_ratio=1.991119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=30 tokens_per_ep_rank=[32261, 32600, 32611, 33096, 33096, 32729, 33096, 32655] avg=32768.0000 max=33096 max_rank=3 min=32261 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.492743 moved_to_replicas=64242 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=31 tokens_per_ep_rank=[69176, 32834, 456, 126121, 46, 380, 43, 33088] avg=32768.0000 max=126121 max_rank=3 min=43 min_rank=6 imbalance_ratio=3.848907 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32834, 32948, 33095, 32551, 31935, 32597, 33088] avg=32768.0000 max=33096 max_rank=0 min=31935 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93025 max_load_reduction_pct=0.737585 moved_to_replicas=129105 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=32 tokens_per_ep_rank=[413, 572, 187, 33450, 64932, 33998, 32700, 95892] avg=32768.0000 max=95892 max_rank=7 min=187 min_rank=2 imbalance_ratio=2.926392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=32 tokens_per_ep_rank=[31892, 32408, 32760, 33096, 33096, 33096, 32700, 33096] avg=32768.0000 max=33096 max_rank=3 min=31892 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62796 max_load_reduction_pct=0.654862 moved_to_replicas=95888 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=33 tokens_per_ep_rank=[129766, 216, 31, 34725, 32607, 31640, 32803, 356] avg=32768.0000 max=129766 max_rank=0 min=31 min_rank=2 imbalance_ratio=3.960144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32727, 32564, 33096, 32607, 33096, 32803, 32155] avg=32768.0000 max=33096 max_rank=0 min=32155 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96670 max_load_reduction_pct=0.744956 moved_to_replicas=98299 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=34 tokens_per_ep_rank=[107008, 261, 486, 397, 290, 55750, 65291, 32661] avg=32768.0000 max=107008 max_rank=0 min=261 min_rank=1 imbalance_ratio=3.265625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=34 tokens_per_ep_rank=[33097, 32832, 32681, 31863, 32819, 33095, 33096, 32661] avg=32768.0000 max=33097 max_rank=0 min=31863 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73911 max_load_reduction_pct=0.690705 moved_to_replicas=128761 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=35 tokens_per_ep_rank=[1332, 42819, 32632, 2315, 64221, 148, 53490, 65187] avg=32768.0000 max=65187 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.989349 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=35 tokens_per_ep_rank=[32457, 33094, 32632, 32434, 33096, 32239, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32239 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32091 max_load_reduction_pct=0.492291 moved_to_replicas=93333 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=36 tokens_per_ep_rank=[91764, 34050, 403, 5132, 32727, 32767, 32580, 32721] avg=32768.0000 max=91764 max_rank=0 min=403 min_rank=2 imbalance_ratio=2.800415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32877, 32280, 32727, 32767, 32580, 32721] avg=32768.0000 max=33096 max_rank=0 min=32280 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58668 max_load_reduction_pct=0.639336 moved_to_replicas=59622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=37 tokens_per_ep_rank=[551, 64649, 33500, 32988, 32814, 103, 32454, 65085] avg=32768.0000 max=65085 max_rank=7 min=103 min_rank=5 imbalance_ratio=1.986237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32988, 32814, 32496, 32454, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31989 max_load_reduction_pct=0.491496 moved_to_replicas=63946 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=38 tokens_per_ep_rank=[3787, 32985, 32621, 61842, 256, 32505, 32723, 65425] avg=32768.0000 max=65425 max_rank=7 min=256 min_rank=4 imbalance_ratio=1.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32985, 32621, 33096, 32585, 32505, 32723, 33096] avg=32768.0000 max=33096 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32329 max_load_reduction_pct=0.494138 moved_to_replicas=61075 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=39 tokens_per_ep_rank=[97582, 165, 32716, 662, 32877, 561, 62223, 35358] avg=32768.0000 max=97582 max_rank=0 min=165 min_rank=1 imbalance_ratio=2.977966 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32613, 32716, 32051, 32877, 32599, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64486 max_load_reduction_pct=0.660839 moved_to_replicas=95875 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1108, 33098, 65342, 481, 65120, 65180, 226] avg=32768.0000 max=65342 max_rank=3 min=226 min_rank=7 imbalance_ratio=1.994080 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=40 tokens_per_ep_rank=[31627, 33096, 33096, 33096, 32565, 33096, 33096, 32472] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32246 max_load_reduction_pct=0.493496 moved_to_replicas=96356 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=41 tokens_per_ep_rank=[409, 65249, 396, 32632, 65048, 32245, 787, 65378] avg=32768.0000 max=65378 max_rank=7 min=396 min_rank=2 imbalance_ratio=1.995178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32678, 32632, 33096, 32245, 32739, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=96387 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=42 tokens_per_ep_rank=[65426, 30850, 380, 129904, 258, 277, 2626, 32423] avg=32768.0000 max=129904 max_rank=3 min=258 min_rank=4 imbalance_ratio=3.964355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32710, 32154, 33096, 32796, 32773, 33096, 32423] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96808 max_load_reduction_pct=0.745227 moved_to_replicas=129138 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 30382, 253, 38606, 32580, 62950, 31397] avg=32768.0000 max=62950 max_rank=6 min=253 min_rank=3 imbalance_ratio=1.921082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=43 tokens_per_ep_rank=[33024, 32952, 32903, 33096, 33096, 32580, 33096, 31397] avg=32768.0000 max=33096 max_rank=3 min=31397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29854 max_load_reduction_pct=0.474249 moved_to_replicas=35364 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=44 tokens_per_ep_rank=[39530, 89942, 159, 64869, 383, 66595, 538, 128] avg=32768.0000 max=89942 max_rank=1 min=128 min_rank=7 imbalance_ratio=2.744812 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=44 tokens_per_ep_rank=[33097, 33098, 31237, 33096, 32844, 33096, 33096, 32580] avg=32768.0000 max=33098 max_rank=1 min=31237 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56844 max_load_reduction_pct=0.632007 moved_to_replicas=128552 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=45 tokens_per_ep_rank=[32120, 64848, 32425, 228, 65450, 32558, 33484, 1031] avg=32768.0000 max=65450 max_rank=4 min=228 min_rank=3 imbalance_ratio=1.997375 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=45 tokens_per_ep_rank=[32508, 33096, 32425, 32582, 33096, 32558, 33096, 32783] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64494 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=46 tokens_per_ep_rank=[527, 33677, 64924, 31815, 32867, 36635, 154, 61545] avg=32768.0000 max=64924 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=46 tokens_per_ep_rank=[32513, 33096, 33096, 32396, 32867, 33096, 31982, 33098] avg=32768.0000 max=33098 max_rank=7 min=31982 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=31826 max_load_reduction_pct=0.490204 moved_to_replicas=64397 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=47 tokens_per_ep_rank=[298, 33011, 276, 33029, 32687, 65459, 62386, 34998] avg=32768.0000 max=65459 max_rank=5 min=276 min_rank=2 imbalance_ratio=1.997650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=47 tokens_per_ep_rank=[31490, 33011, 32639, 33029, 32687, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31490 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32363 max_load_reduction_pct=0.494401 moved_to_replicas=63555 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=48 tokens_per_ep_rank=[2500, 32722, 30501, 269, 97657, 507, 64759, 33229] avg=32768.0000 max=97657 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980255 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32722, 31701, 32817, 33096, 32520, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31701 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64561 max_load_reduction_pct=0.661100 moved_to_replicas=96357 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=48 total_expected_recv=1574944 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365184 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=48 total_expected_recv=1572404 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368512 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=48 total_expected_recv=1574820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364800 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=48 total_expected_recv=1571340 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=367872 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=48 total_expected_recv=1569201 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=371328 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=48 total_expected_recv=1571289 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366336 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=48 total_expected_recv=1574339 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=367616 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=48 total_expected_recv=1574575 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364416 peak_demand=33098 required_capacity=1.010071 - [2026-08-26 15:01:21] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 719.4 | throughput per GPU (TFLOP/s/GPU): 123.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604043E+00 | loss scale: 1.0 | grad norm: 8.752 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=31141 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=31141 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=46 calls=1 budget=40960 expected_recv=33060 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[39385, 21239, 32560, 21759, 45306, 25241, 43122, 33532] avg=32768.0000 max=45306 max_rank=4 min=21239 min_rank=1 imbalance_ratio=1.382629 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[8493, 32627, 35048, 174, 18928, 94651, 22897, 49326] avg=32768.0000 max=94651 max_rank=5 min=174 min_rank=3 imbalance_ratio=2.888519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31008 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=12208 max_load_reduction_pct=0.269457 moved_to_replicas=28961 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31008 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=12208 max_load_reduction_pct=0.269457 moved_to_replicas=28961 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[8493, 32627, 35048, 174, 18928, 94651, 22897, 49326] avg=32768.0000 max=94651 max_rank=5 min=174 min_rank=3 imbalance_ratio=2.888519 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=47 calls=1 budget=40960 expected_recv=31708 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31008 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=12208 max_load_reduction_pct=0.269457 moved_to_replicas=28961 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[39385, 21239, 32560, 21759, 45306, 25241, 43122, 33532] avg=32768.0000 max=45306 max_rank=4 min=21239 min_rank=1 imbalance_ratio=1.382629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] avg=32768.0000 max=33097 max_rank=6 min=31486 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61554 max_load_reduction_pct=0.650326 moved_to_replicas=79737 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=47 calls=1 budget=40960 expected_recv=31708 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] avg=32768.0000 max=33097 max_rank=6 min=31486 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61554 max_load_reduction_pct=0.650326 moved_to_replicas=79737 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[8493, 32627, 35048, 174, 18928, 94651, 22897, 49326] avg=32768.0000 max=94651 max_rank=5 min=174 min_rank=3 imbalance_ratio=2.888519 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[8493, 32627, 35048, 174, 18928, 94651, 22897, 49326] avg=32768.0000 max=94651 max_rank=5 min=174 min_rank=3 imbalance_ratio=2.888519 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31008 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=12208 max_load_reduction_pct=0.269457 moved_to_replicas=28961 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[8493, 32627, 35048, 174, 18928, 94651, 22897, 49326] avg=32768.0000 max=94651 max_rank=5 min=174 min_rank=3 imbalance_ratio=2.888519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[17649, 38190, 32405, 62912, 66319, 148, 1291, 43230] avg=32768.0000 max=66319 max_rank=4 min=148 min_rank=5 imbalance_ratio=2.023895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] avg=32768.0000 max=33097 max_rank=6 min=31486 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61554 max_load_reduction_pct=0.650326 moved_to_replicas=79737 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] avg=32768.0000 max=33097 max_rank=6 min=31486 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61554 max_load_reduction_pct=0.650326 moved_to_replicas=79737 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[17649, 38190, 32405, 62912, 66319, 148, 1291, 43230] avg=32768.0000 max=66319 max_rank=4 min=148 min_rank=5 imbalance_ratio=2.023895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[39385, 21239, 32560, 21759, 45306, 25241, 43122, 33532] avg=32768.0000 max=45306 max_rank=4 min=21239 min_rank=1 imbalance_ratio=1.382629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] avg=32768.0000 max=33097 max_rank=6 min=31486 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61554 max_load_reduction_pct=0.650326 moved_to_replicas=79737 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] avg=32768.0000 max=33096 max_rank=1 min=31615 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33223 max_load_reduction_pct=0.500957 moved_to_replicas=78267 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[8493, 32627, 35048, 174, 18928, 94651, 22897, 49326] avg=32768.0000 max=94651 max_rank=5 min=174 min_rank=3 imbalance_ratio=2.888519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[39385, 21239, 32560, 21759, 45306, 25241, 43122, 33532] avg=32768.0000 max=45306 max_rank=4 min=21239 min_rank=1 imbalance_ratio=1.382629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] avg=32768.0000 max=33096 max_rank=1 min=31615 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33223 max_load_reduction_pct=0.500957 moved_to_replicas=78267 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[17649, 38190, 32405, 62912, 66319, 148, 1291, 43230] avg=32768.0000 max=66319 max_rank=4 min=148 min_rank=5 imbalance_ratio=2.023895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31008 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=12208 max_load_reduction_pct=0.269457 moved_to_replicas=28961 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[17649, 38190, 32405, 62912, 66319, 148, 1291, 43230] avg=32768.0000 max=66319 max_rank=4 min=148 min_rank=5 imbalance_ratio=2.023895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] avg=32768.0000 max=33097 max_rank=6 min=31486 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61554 max_load_reduction_pct=0.650326 moved_to_replicas=79737 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[17649, 38190, 32405, 62912, 66319, 148, 1291, 43230] avg=32768.0000 max=66319 max_rank=4 min=148 min_rank=5 imbalance_ratio=2.023895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[33095, 33096, 32560, 31008, 33096, 33098, 33095, 33096] avg=32768.0000 max=33098 max_rank=5 min=31008 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=12208 max_load_reduction_pct=0.269457 moved_to_replicas=28961 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[46, 32207, 45270, 70518, 61, 32802, 43006, 38234] avg=32768.0000 max=70518 max_rank=3 min=46 min_rank=0 imbalance_ratio=2.152039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] avg=32768.0000 max=33096 max_rank=1 min=31615 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33223 max_load_reduction_pct=0.500957 moved_to_replicas=78267 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] avg=32768.0000 max=33096 max_rank=1 min=31615 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33223 max_load_reduction_pct=0.500957 moved_to_replicas=78267 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[46, 32207, 45270, 70518, 61, 32802, 43006, 38234] avg=32768.0000 max=70518 max_rank=3 min=46 min_rank=0 imbalance_ratio=2.152039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] avg=32768.0000 max=33096 max_rank=1 min=31615 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33223 max_load_reduction_pct=0.500957 moved_to_replicas=78267 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[8493, 32627, 35048, 174, 18928, 94651, 22897, 49326] avg=32768.0000 max=94651 max_rank=5 min=174 min_rank=3 imbalance_ratio=2.888519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[17649, 38190, 32405, 62912, 66319, 148, 1291, 43230] avg=32768.0000 max=66319 max_rank=4 min=148 min_rank=5 imbalance_ratio=2.023895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31978 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37421 max_load_reduction_pct=0.530659 moved_to_replicas=64644 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31978 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37421 max_load_reduction_pct=0.530659 moved_to_replicas=64644 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[46, 32207, 45270, 70518, 61, 32802, 43006, 38234] avg=32768.0000 max=70518 max_rank=3 min=46 min_rank=0 imbalance_ratio=2.152039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[8493, 32627, 35048, 174, 18928, 94651, 22897, 49326] avg=32768.0000 max=94651 max_rank=5 min=174 min_rank=3 imbalance_ratio=2.888519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[46, 32207, 45270, 70518, 61, 32802, 43006, 38234] avg=32768.0000 max=70518 max_rank=3 min=46 min_rank=0 imbalance_ratio=2.152039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] avg=32768.0000 max=33097 max_rank=6 min=31486 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61554 max_load_reduction_pct=0.650326 moved_to_replicas=79737 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] avg=32768.0000 max=33096 max_rank=1 min=31615 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33223 max_load_reduction_pct=0.500957 moved_to_replicas=78267 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[46, 32207, 45270, 70518, 61, 32802, 43006, 38234] avg=32768.0000 max=70518 max_rank=3 min=46 min_rank=0 imbalance_ratio=2.152039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31978 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37421 max_load_reduction_pct=0.530659 moved_to_replicas=64644 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[65691, 2020, 74252, 54651, 31871, 122, 2392, 31145] avg=32768.0000 max=74252 max_rank=2 min=122 min_rank=5 imbalance_ratio=2.265991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32627, 33096, 32552, 31486, 33096, 33097, 33094] avg=32768.0000 max=33097 max_rank=6 min=31486 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61554 max_load_reduction_pct=0.650326 moved_to_replicas=79737 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31978 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37421 max_load_reduction_pct=0.530659 moved_to_replicas=64644 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[65691, 2020, 74252, 54651, 31871, 122, 2392, 31145] avg=32768.0000 max=74252 max_rank=2 min=122 min_rank=5 imbalance_ratio=2.265991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31978 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37421 max_load_reduction_pct=0.530659 moved_to_replicas=64644 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] avg=32768.0000 max=33098 max_rank=1 min=31871 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=41154 max_load_reduction_pct=0.554248 moved_to_replicas=95306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[17649, 38190, 32405, 62912, 66319, 148, 1291, 43230] avg=32768.0000 max=66319 max_rank=4 min=148 min_rank=5 imbalance_ratio=2.023895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[46, 32207, 45270, 70518, 61, 32802, 43006, 38234] avg=32768.0000 max=70518 max_rank=3 min=46 min_rank=0 imbalance_ratio=2.152039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[65691, 2020, 74252, 54651, 31871, 122, 2392, 31145] avg=32768.0000 max=74252 max_rank=2 min=122 min_rank=5 imbalance_ratio=2.265991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] avg=32768.0000 max=33098 max_rank=1 min=31871 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=41154 max_load_reduction_pct=0.554248 moved_to_replicas=95306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[17649, 38190, 32405, 62912, 66319, 148, 1291, 43230] avg=32768.0000 max=66319 max_rank=4 min=148 min_rank=5 imbalance_ratio=2.023895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[65691, 2020, 74252, 54651, 31871, 122, 2392, 31145] avg=32768.0000 max=74252 max_rank=2 min=122 min_rank=5 imbalance_ratio=2.265991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] avg=32768.0000 max=33096 max_rank=1 min=31615 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33223 max_load_reduction_pct=0.500957 moved_to_replicas=78267 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31978 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37421 max_load_reduction_pct=0.530659 moved_to_replicas=64644 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[65691, 2020, 74252, 54651, 31871, 122, 2392, 31145] avg=32768.0000 max=74252 max_rank=2 min=122 min_rank=5 imbalance_ratio=2.265991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] avg=32768.0000 max=33098 max_rank=1 min=31871 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=41154 max_load_reduction_pct=0.554248 moved_to_replicas=95306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[3469, 33711, 166, 65041, 29664, 100, 41268, 88725] avg=32768.0000 max=88725 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.707672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32878, 33096, 32405, 33096, 33096, 32863, 31615, 33095] avg=32768.0000 max=33096 max_rank=1 min=31615 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33223 max_load_reduction_pct=0.500957 moved_to_replicas=78267 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] avg=32768.0000 max=33098 max_rank=1 min=31871 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=41154 max_load_reduction_pct=0.554248 moved_to_replicas=95306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[3469, 33711, 166, 65041, 29664, 100, 41268, 88725] avg=32768.0000 max=88725 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.707672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] avg=32768.0000 max=33098 max_rank=1 min=31871 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=41154 max_load_reduction_pct=0.554248 moved_to_replicas=95306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30764 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=55628 max_load_reduction_pct=0.626971 moved_to_replicas=96361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[46, 32207, 45270, 70518, 61, 32802, 43006, 38234] avg=32768.0000 max=70518 max_rank=3 min=46 min_rank=0 imbalance_ratio=2.152039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[65691, 2020, 74252, 54651, 31871, 122, 2392, 31145] avg=32768.0000 max=74252 max_rank=2 min=122 min_rank=5 imbalance_ratio=2.265991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[3469, 33711, 166, 65041, 29664, 100, 41268, 88725] avg=32768.0000 max=88725 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.707672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30764 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=55628 max_load_reduction_pct=0.626971 moved_to_replicas=96361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[46, 32207, 45270, 70518, 61, 32802, 43006, 38234] avg=32768.0000 max=70518 max_rank=3 min=46 min_rank=0 imbalance_ratio=2.152039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[3469, 33711, 166, 65041, 29664, 100, 41268, 88725] avg=32768.0000 max=88725 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.707672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] avg=32768.0000 max=33098 max_rank=1 min=31871 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=41154 max_load_reduction_pct=0.554248 moved_to_replicas=95306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31978 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37421 max_load_reduction_pct=0.530659 moved_to_replicas=64644 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30764 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=55628 max_load_reduction_pct=0.626971 moved_to_replicas=96361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[43238, 32771, 230, 33231, 51, 51327, 22311, 78985] avg=32768.0000 max=78985 max_rank=7 min=51 min_rank=4 imbalance_ratio=2.410431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[3469, 33711, 166, 65041, 29664, 100, 41268, 88725] avg=32768.0000 max=88725 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.707672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32772, 32207, 33096, 33097, 31978, 32802, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31978 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37421 max_load_reduction_pct=0.530659 moved_to_replicas=64644 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30764 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=55628 max_load_reduction_pct=0.626971 moved_to_replicas=96361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[43238, 32771, 230, 33231, 51, 51327, 22311, 78985] avg=32768.0000 max=78985 max_rank=7 min=51 min_rank=4 imbalance_ratio=2.410431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] avg=32768.0000 max=33097 max_rank=5 min=31787 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45888 max_load_reduction_pct=0.580971 moved_to_replicas=74397 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30764 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=55628 max_load_reduction_pct=0.626971 moved_to_replicas=96361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[3469, 33711, 166, 65041, 29664, 100, 41268, 88725] avg=32768.0000 max=88725 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.707672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[65691, 2020, 74252, 54651, 31871, 122, 2392, 31145] avg=32768.0000 max=74252 max_rank=2 min=122 min_rank=5 imbalance_ratio=2.265991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[43238, 32771, 230, 33231, 51, 51327, 22311, 78985] avg=32768.0000 max=78985 max_rank=7 min=51 min_rank=4 imbalance_ratio=2.410431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] avg=32768.0000 max=33097 max_rank=5 min=31787 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45888 max_load_reduction_pct=0.580971 moved_to_replicas=74397 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[65691, 2020, 74252, 54651, 31871, 122, 2392, 31145] avg=32768.0000 max=74252 max_rank=2 min=122 min_rank=5 imbalance_ratio=2.265991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[43238, 32771, 230, 33231, 51, 51327, 22311, 78985] avg=32768.0000 max=78985 max_rank=7 min=51 min_rank=4 imbalance_ratio=2.410431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30764 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=55628 max_load_reduction_pct=0.626971 moved_to_replicas=96361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] avg=32768.0000 max=33098 max_rank=1 min=31871 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=41154 max_load_reduction_pct=0.554248 moved_to_replicas=95306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] avg=32768.0000 max=33097 max_rank=5 min=31787 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45888 max_load_reduction_pct=0.580971 moved_to_replicas=74397 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[30525, 267, 25395, 62007, 24, 65541, 31668, 46717] avg=32768.0000 max=65541 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[43238, 32771, 230, 33231, 51, 51327, 22311, 78985] avg=32768.0000 max=78985 max_rank=7 min=51 min_rank=4 imbalance_ratio=2.410431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[33096, 33098, 33096, 33094, 31871, 32825, 33096, 31968] avg=32768.0000 max=33098 max_rank=1 min=31871 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=41154 max_load_reduction_pct=0.554248 moved_to_replicas=95306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] avg=32768.0000 max=33097 max_rank=5 min=31787 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45888 max_load_reduction_pct=0.580971 moved_to_replicas=74397 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[30525, 267, 25395, 62007, 24, 65541, 31668, 46717] avg=32768.0000 max=65541 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] avg=32768.0000 max=33097 max_rank=7 min=31668 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32444 max_load_reduction_pct=0.495018 moved_to_replicas=74977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] avg=32768.0000 max=33097 max_rank=5 min=31787 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45888 max_load_reduction_pct=0.580971 moved_to_replicas=74397 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[43238, 32771, 230, 33231, 51, 51327, 22311, 78985] avg=32768.0000 max=78985 max_rank=7 min=51 min_rank=4 imbalance_ratio=2.410431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] avg=32768.0000 max=33097 max_rank=7 min=31668 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32444 max_load_reduction_pct=0.495018 moved_to_replicas=74977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[30525, 267, 25395, 62007, 24, 65541, 31668, 46717] avg=32768.0000 max=65541 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[3469, 33711, 166, 65041, 29664, 100, 41268, 88725] avg=32768.0000 max=88725 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.707672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[30525, 267, 25395, 62007, 24, 65541, 31668, 46717] avg=32768.0000 max=65541 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[3469, 33711, 166, 65041, 29664, 100, 41268, 88725] avg=32768.0000 max=88725 max_rank=7 min=100 min_rank=5 imbalance_ratio=2.707672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] avg=32768.0000 max=33097 max_rank=5 min=31787 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45888 max_load_reduction_pct=0.580971 moved_to_replicas=74397 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] avg=32768.0000 max=33097 max_rank=7 min=31668 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32444 max_load_reduction_pct=0.495018 moved_to_replicas=74977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[64976, 322, 2333, 63366, 32793, 32766, 32847, 32741] avg=32768.0000 max=64976 max_rank=0 min=322 min_rank=1 imbalance_ratio=1.982910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[30525, 267, 25395, 62007, 24, 65541, 31668, 46717] avg=32768.0000 max=65541 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30764 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=55628 max_load_reduction_pct=0.626971 moved_to_replicas=96361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[64976, 322, 2333, 63366, 32793, 32766, 32847, 32741] avg=32768.0000 max=64976 max_rank=0 min=322 min_rank=1 imbalance_ratio=1.982910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] avg=32768.0000 max=33097 max_rank=7 min=31668 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32444 max_load_reduction_pct=0.495018 moved_to_replicas=74977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30764, 32804, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=30764 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=55628 max_load_reduction_pct=0.626971 moved_to_replicas=96361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] avg=32768.0000 max=33096 max_rank=0 min=32202 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31880 max_load_reduction_pct=0.490643 moved_to_replicas=62150 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] avg=32768.0000 max=33097 max_rank=7 min=31668 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32444 max_load_reduction_pct=0.495018 moved_to_replicas=74977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[30525, 267, 25395, 62007, 24, 65541, 31668, 46717] avg=32768.0000 max=65541 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] avg=32768.0000 max=33096 max_rank=0 min=32202 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31880 max_load_reduction_pct=0.490643 moved_to_replicas=62150 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[64976, 322, 2333, 63366, 32793, 32766, 32847, 32741] avg=32768.0000 max=64976 max_rank=0 min=322 min_rank=1 imbalance_ratio=1.982910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[43238, 32771, 230, 33231, 51, 51327, 22311, 78985] avg=32768.0000 max=78985 max_rank=7 min=51 min_rank=4 imbalance_ratio=2.410431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[64976, 322, 2333, 63366, 32793, 32766, 32847, 32741] avg=32768.0000 max=64976 max_rank=0 min=322 min_rank=1 imbalance_ratio=1.982910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[43238, 32771, 230, 33231, 51, 51327, 22311, 78985] avg=32768.0000 max=78985 max_rank=7 min=51 min_rank=4 imbalance_ratio=2.410431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] avg=32768.0000 max=33097 max_rank=7 min=31668 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32444 max_load_reduction_pct=0.495018 moved_to_replicas=74977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] avg=32768.0000 max=33096 max_rank=0 min=32202 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31880 max_load_reduction_pct=0.490643 moved_to_replicas=62150 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 4557, 89796, 263, 65497, 3995] avg=32768.0000 max=89796 max_rank=4 min=263 min_rank=5 imbalance_ratio=2.740356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[64976, 322, 2333, 63366, 32793, 32766, 32847, 32741] avg=32768.0000 max=64976 max_rank=0 min=322 min_rank=1 imbalance_ratio=1.982910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] avg=32768.0000 max=33097 max_rank=5 min=31787 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45888 max_load_reduction_pct=0.580971 moved_to_replicas=74397 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 4557, 89796, 263, 65497, 3995] avg=32768.0000 max=89796 max_rank=4 min=263 min_rank=5 imbalance_ratio=2.740356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] avg=32768.0000 max=33096 max_rank=0 min=32202 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31880 max_load_reduction_pct=0.490643 moved_to_replicas=62150 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[33096, 32771, 31787, 33096, 32750, 33097, 32453, 33094] avg=32768.0000 max=33097 max_rank=5 min=31787 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=45888 max_load_reduction_pct=0.580971 moved_to_replicas=74397 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] avg=32768.0000 max=33096 max_rank=3 min=31882 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.631431 moved_to_replicas=89101 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] avg=32768.0000 max=33096 max_rank=0 min=32202 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31880 max_load_reduction_pct=0.490643 moved_to_replicas=62150 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] avg=32768.0000 max=33096 max_rank=3 min=31882 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.631431 moved_to_replicas=89101 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[64976, 322, 2333, 63366, 32793, 32766, 32847, 32741] avg=32768.0000 max=64976 max_rank=0 min=322 min_rank=1 imbalance_ratio=1.982910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 4557, 89796, 263, 65497, 3995] avg=32768.0000 max=89796 max_rank=4 min=263 min_rank=5 imbalance_ratio=2.740356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[30525, 267, 25395, 62007, 24, 65541, 31668, 46717] avg=32768.0000 max=65541 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 4557, 89796, 263, 65497, 3995] avg=32768.0000 max=89796 max_rank=4 min=263 min_rank=5 imbalance_ratio=2.740356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[30525, 267, 25395, 62007, 24, 65541, 31668, 46717] avg=32768.0000 max=65541 max_rank=5 min=24 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] avg=32768.0000 max=33096 max_rank=0 min=32202 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31880 max_load_reduction_pct=0.490643 moved_to_replicas=62150 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[219, 1358, 44838, 32007, 52789, 302, 98007, 32624] avg=32768.0000 max=98007 max_rank=6 min=219 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] avg=32768.0000 max=33096 max_rank=3 min=31882 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.631431 moved_to_replicas=89101 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 4557, 89796, 263, 65497, 3995] avg=32768.0000 max=89796 max_rank=4 min=263 min_rank=5 imbalance_ratio=2.740356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[219, 1358, 44838, 32007, 52789, 302, 98007, 32624] avg=32768.0000 max=98007 max_rank=6 min=219 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] avg=32768.0000 max=33097 max_rank=7 min=31668 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32444 max_load_reduction_pct=0.495018 moved_to_replicas=74977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] avg=32768.0000 max=33096 max_rank=3 min=31882 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.631431 moved_to_replicas=89101 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32527, 33095, 33096, 33096, 32469, 33096, 31668, 33097] avg=32768.0000 max=33097 max_rank=7 min=31668 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32444 max_load_reduction_pct=0.495018 moved_to_replicas=74977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] avg=32768.0000 max=33096 max_rank=4 min=32007 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96346 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] avg=32768.0000 max=33096 max_rank=3 min=31882 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.631431 moved_to_replicas=89101 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] avg=32768.0000 max=33096 max_rank=4 min=32007 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96346 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 4557, 89796, 263, 65497, 3995] avg=32768.0000 max=89796 max_rank=4 min=263 min_rank=5 imbalance_ratio=2.740356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[219, 1358, 44838, 32007, 52789, 302, 98007, 32624] avg=32768.0000 max=98007 max_rank=6 min=219 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[64976, 322, 2333, 63366, 32793, 32766, 32847, 32741] avg=32768.0000 max=64976 max_rank=0 min=322 min_rank=1 imbalance_ratio=1.982910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[219, 1358, 44838, 32007, 52789, 302, 98007, 32624] avg=32768.0000 max=98007 max_rank=6 min=219 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[64976, 322, 2333, 63366, 32793, 32766, 32847, 32741] avg=32768.0000 max=64976 max_rank=0 min=322 min_rank=1 imbalance_ratio=1.982910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] avg=32768.0000 max=33096 max_rank=3 min=31882 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.631431 moved_to_replicas=89101 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[167, 76538, 94360, 58216, 31, 32747, 85, 0] avg=32768.0000 max=94360 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.879639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[219, 1358, 44838, 32007, 52789, 302, 98007, 32624] avg=32768.0000 max=98007 max_rank=6 min=219 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] avg=32768.0000 max=33096 max_rank=4 min=32007 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96346 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] avg=32768.0000 max=33096 max_rank=0 min=32202 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31880 max_load_reduction_pct=0.490643 moved_to_replicas=62150 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[167, 76538, 94360, 58216, 31, 32747, 85, 0] avg=32768.0000 max=94360 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.879639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] avg=32768.0000 max=33096 max_rank=4 min=32007 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96346 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32202, 32603, 33096, 32793, 32766, 32847, 32741] avg=32768.0000 max=33096 max_rank=0 min=32202 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31880 max_load_reduction_pct=0.490643 moved_to_replicas=62150 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] avg=32768.0000 max=33097 max_rank=1 min=31586 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61263 max_load_reduction_pct=0.649248 moved_to_replicas=129826 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] avg=32768.0000 max=33096 max_rank=4 min=32007 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96346 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[219, 1358, 44838, 32007, 52789, 302, 98007, 32624] avg=32768.0000 max=98007 max_rank=6 min=219 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] avg=32768.0000 max=33097 max_rank=1 min=31586 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61263 max_load_reduction_pct=0.649248 moved_to_replicas=129826 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 4557, 89796, 263, 65497, 3995] avg=32768.0000 max=89796 max_rank=4 min=263 min_rank=5 imbalance_ratio=2.740356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[167, 76538, 94360, 58216, 31, 32747, 85, 0] avg=32768.0000 max=94360 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.879639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[167, 76538, 94360, 58216, 31, 32747, 85, 0] avg=32768.0000 max=94360 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.879639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 4557, 89796, 263, 65497, 3995] avg=32768.0000 max=89796 max_rank=4 min=263 min_rank=5 imbalance_ratio=2.740356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] avg=32768.0000 max=33096 max_rank=4 min=32007 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96346 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[2472, 33496, 143, 65344, 42434, 98019, 207, 20029] avg=32768.0000 max=98019 max_rank=5 min=143 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[167, 76538, 94360, 58216, 31, 32747, 85, 0] avg=32768.0000 max=94360 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.879639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] avg=32768.0000 max=33096 max_rank=3 min=31882 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.631431 moved_to_replicas=89101 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[2472, 33496, 143, 65344, 42434, 98019, 207, 20029] avg=32768.0000 max=98019 max_rank=5 min=143 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] avg=32768.0000 max=33097 max_rank=1 min=31586 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61263 max_load_reduction_pct=0.649248 moved_to_replicas=129826 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] avg=32768.0000 max=33097 max_rank=1 min=31586 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61263 max_load_reduction_pct=0.649248 moved_to_replicas=129826 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32687, 32668, 32681, 33096, 33095, 32939, 33096, 31882] avg=32768.0000 max=33096 max_rank=3 min=31882 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.631431 moved_to_replicas=89101 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] avg=32768.0000 max=33096 max_rank=0 min=31391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] avg=32768.0000 max=33097 max_rank=1 min=31586 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61263 max_load_reduction_pct=0.649248 moved_to_replicas=129826 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] avg=32768.0000 max=33096 max_rank=0 min=31391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[167, 76538, 94360, 58216, 31, 32747, 85, 0] avg=32768.0000 max=94360 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.879639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[219, 1358, 44838, 32007, 52789, 302, 98007, 32624] avg=32768.0000 max=98007 max_rank=6 min=219 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[2472, 33496, 143, 65344, 42434, 98019, 207, 20029] avg=32768.0000 max=98019 max_rank=5 min=143 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[2472, 33496, 143, 65344, 42434, 98019, 207, 20029] avg=32768.0000 max=98019 max_rank=5 min=143 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32804, 65329, 97026, 3786, 83, 120, 33053, 29943] avg=32768.0000 max=97026 max_rank=2 min=83 min_rank=4 imbalance_ratio=2.960999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] avg=32768.0000 max=33097 max_rank=1 min=31586 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61263 max_load_reduction_pct=0.649248 moved_to_replicas=129826 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[219, 1358, 44838, 32007, 52789, 302, 98007, 32624] avg=32768.0000 max=98007 max_rank=6 min=219 min_rank=0 imbalance_ratio=2.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[2472, 33496, 143, 65344, 42434, 98019, 207, 20029] avg=32768.0000 max=98019 max_rank=5 min=143 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32804, 65329, 97026, 3786, 83, 120, 33053, 29943] avg=32768.0000 max=97026 max_rank=2 min=83 min_rank=4 imbalance_ratio=2.960999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] avg=32768.0000 max=33096 max_rank=4 min=32007 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96346 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] avg=32768.0000 max=33096 max_rank=0 min=31391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] avg=32768.0000 max=33096 max_rank=0 min=31391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] avg=32768.0000 max=33096 max_rank=1 min=31621 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63930 max_load_reduction_pct=0.658896 moved_to_replicas=96163 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32911, 32794, 33095, 32007, 33096, 32521, 33096, 32624] avg=32768.0000 max=33096 max_rank=4 min=32007 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64911 max_load_reduction_pct=0.662310 moved_to_replicas=96346 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] avg=32768.0000 max=33096 max_rank=0 min=31391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] avg=32768.0000 max=33096 max_rank=1 min=31621 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63930 max_load_reduction_pct=0.658896 moved_to_replicas=96163 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[2472, 33496, 143, 65344, 42434, 98019, 207, 20029] avg=32768.0000 max=98019 max_rank=5 min=143 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32804, 65329, 97026, 3786, 83, 120, 33053, 29943] avg=32768.0000 max=97026 max_rank=2 min=83 min_rank=4 imbalance_ratio=2.960999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32804, 65329, 97026, 3786, 83, 120, 33053, 29943] avg=32768.0000 max=97026 max_rank=2 min=83 min_rank=4 imbalance_ratio=2.960999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[167, 76538, 94360, 58216, 31, 32747, 85, 0] avg=32768.0000 max=94360 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.879639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[31763, 65450, 65199, 1908, 32748, 95, 32657, 32324] avg=32768.0000 max=65450 max_rank=1 min=95 min_rank=5 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] avg=32768.0000 max=33096 max_rank=0 min=31391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[167, 76538, 94360, 58216, 31, 32747, 85, 0] avg=32768.0000 max=94360 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.879639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32804, 65329, 97026, 3786, 83, 120, 33053, 29943] avg=32768.0000 max=97026 max_rank=2 min=83 min_rank=4 imbalance_ratio=2.960999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[31763, 65450, 65199, 1908, 32748, 95, 32657, 32324] avg=32768.0000 max=65450 max_rank=1 min=95 min_rank=5 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] avg=32768.0000 max=33096 max_rank=1 min=31621 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63930 max_load_reduction_pct=0.658896 moved_to_replicas=96163 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] avg=32768.0000 max=33096 max_rank=1 min=31621 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63930 max_load_reduction_pct=0.658896 moved_to_replicas=96163 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] avg=32768.0000 max=33097 max_rank=1 min=31586 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61263 max_load_reduction_pct=0.649248 moved_to_replicas=129826 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] avg=32768.0000 max=33096 max_rank=1 min=32324 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64457 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[33094, 33097, 33096, 33097, 31586, 32747, 32762, 32665] avg=32768.0000 max=33097 max_rank=1 min=31586 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=61263 max_load_reduction_pct=0.649248 moved_to_replicas=129826 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] avg=32768.0000 max=33096 max_rank=1 min=32324 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64457 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] avg=32768.0000 max=33096 max_rank=1 min=31621 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63930 max_load_reduction_pct=0.658896 moved_to_replicas=96163 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32804, 65329, 97026, 3786, 83, 120, 33053, 29943] avg=32768.0000 max=97026 max_rank=2 min=83 min_rank=4 imbalance_ratio=2.960999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[31763, 65450, 65199, 1908, 32748, 95, 32657, 32324] avg=32768.0000 max=65450 max_rank=1 min=95 min_rank=5 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[31763, 65450, 65199, 1908, 32748, 95, 32657, 32324] avg=32768.0000 max=65450 max_rank=1 min=95 min_rank=5 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[2472, 33496, 143, 65344, 42434, 98019, 207, 20029] avg=32768.0000 max=98019 max_rank=5 min=143 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[65179, 32778, 32821, 32661, 65732, 474, 32492, 7] avg=32768.0000 max=65732 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] avg=32768.0000 max=33096 max_rank=1 min=31621 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63930 max_load_reduction_pct=0.658896 moved_to_replicas=96163 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[65179, 32778, 32821, 32661, 65732, 474, 32492, 7] avg=32768.0000 max=65732 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[2472, 33496, 143, 65344, 42434, 98019, 207, 20029] avg=32768.0000 max=98019 max_rank=5 min=143 min_rank=2 imbalance_ratio=2.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[31763, 65450, 65199, 1908, 32748, 95, 32657, 32324] avg=32768.0000 max=65450 max_rank=1 min=95 min_rank=5 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] avg=32768.0000 max=33096 max_rank=1 min=32324 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64457 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] avg=32768.0000 max=33096 max_rank=1 min=32324 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64457 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] avg=32768.0000 max=33096 max_rank=0 min=31391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] avg=32768.0000 max=33096 max_rank=0 min=32492 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32636 max_load_reduction_pct=0.496501 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] avg=32768.0000 max=33096 max_rank=0 min=32492 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32636 max_load_reduction_pct=0.496501 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[33096, 33096, 32801, 33096, 33096, 33096, 32472, 31391] avg=32768.0000 max=33096 max_rank=0 min=31391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64923 max_load_reduction_pct=0.662351 moved_to_replicas=106909 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] avg=32768.0000 max=33096 max_rank=1 min=32324 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64457 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[31763, 65450, 65199, 1908, 32748, 95, 32657, 32324] avg=32768.0000 max=65450 max_rank=1 min=95 min_rank=5 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[65179, 32778, 32821, 32661, 65732, 474, 32492, 7] avg=32768.0000 max=65732 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[65179, 32778, 32821, 32661, 65732, 474, 32492, 7] avg=32768.0000 max=65732 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[32787, 20876, 40815, 127, 32671, 65075, 32916, 36877] avg=32768.0000 max=65075 max_rank=5 min=127 min_rank=3 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32804, 65329, 97026, 3786, 83, 120, 33053, 29943] avg=32768.0000 max=97026 max_rank=2 min=83 min_rank=4 imbalance_ratio=2.960999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] avg=32768.0000 max=33096 max_rank=1 min=32324 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64457 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[32787, 20876, 40815, 127, 32671, 65075, 32916, 36877] avg=32768.0000 max=65075 max_rank=5 min=127 min_rank=3 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] avg=32768.0000 max=33096 max_rank=0 min=32492 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32636 max_load_reduction_pct=0.496501 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] avg=32768.0000 max=33096 max_rank=0 min=32492 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32636 max_load_reduction_pct=0.496501 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[65179, 32778, 32821, 32661, 65732, 474, 32492, 7] avg=32768.0000 max=65732 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32804, 65329, 97026, 3786, 83, 120, 33053, 29943] avg=32768.0000 max=97026 max_rank=2 min=83 min_rank=4 imbalance_ratio=2.960999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] avg=32768.0000 max=33096 max_rank=5 min=32106 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=43479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] avg=32768.0000 max=33096 max_rank=1 min=31621 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63930 max_load_reduction_pct=0.658896 moved_to_replicas=96163 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] avg=32768.0000 max=33096 max_rank=5 min=32106 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=43479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[65179, 32778, 32821, 32661, 65732, 474, 32492, 7] avg=32768.0000 max=65732 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] avg=32768.0000 max=33096 max_rank=0 min=32492 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32636 max_load_reduction_pct=0.496501 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32804, 33096, 33096, 33096, 32512, 31621, 33053, 32866] avg=32768.0000 max=33096 max_rank=1 min=31621 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63930 max_load_reduction_pct=0.658896 moved_to_replicas=96163 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[32787, 20876, 40815, 127, 32671, 65075, 32916, 36877] avg=32768.0000 max=65075 max_rank=5 min=127 min_rank=3 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[32787, 20876, 40815, 127, 32671, 65075, 32916, 36877] avg=32768.0000 max=65075 max_rank=5 min=127 min_rank=3 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[131, 80, 135, 65463, 65135, 65395, 33068, 32737] avg=32768.0000 max=65463 max_rank=3 min=80 min_rank=1 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] avg=32768.0000 max=33096 max_rank=0 min=32492 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32636 max_load_reduction_pct=0.496501 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[31763, 65450, 65199, 1908, 32748, 95, 32657, 32324] avg=32768.0000 max=65450 max_rank=1 min=95 min_rank=5 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[131, 80, 135, 65463, 65135, 65395, 33068, 32737] avg=32768.0000 max=65463 max_rank=3 min=80 min_rank=1 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] avg=32768.0000 max=33096 max_rank=5 min=32106 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=43479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] avg=32768.0000 max=33096 max_rank=5 min=32106 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=43479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[32787, 20876, 40815, 127, 32671, 65075, 32916, 36877] avg=32768.0000 max=65075 max_rank=5 min=127 min_rank=3 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[31763, 65450, 65199, 1908, 32748, 95, 32657, 32324] avg=32768.0000 max=65450 max_rank=1 min=95 min_rank=5 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=96705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=96705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] avg=32768.0000 max=33096 max_rank=1 min=32324 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64457 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[32787, 20876, 40815, 127, 32671, 65075, 32916, 36877] avg=32768.0000 max=65075 max_rank=5 min=127 min_rank=3 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] avg=32768.0000 max=33096 max_rank=5 min=32106 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=43479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32678, 33096, 33096, 33096, 32748, 32449, 32657, 32324] avg=32768.0000 max=33096 max_rank=1 min=32324 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=64457 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[131, 80, 135, 65463, 65135, 65395, 33068, 32737] avg=32768.0000 max=65463 max_rank=3 min=80 min_rank=1 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[131, 80, 135, 65463, 65135, 65395, 33068, 32737] avg=32768.0000 max=65463 max_rank=3 min=80 min_rank=1 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[1866, 32574, 114, 65348, 183, 65465, 32673, 63921] avg=32768.0000 max=65465 max_rank=5 min=114 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[65179, 32778, 32821, 32661, 65732, 474, 32492, 7] avg=32768.0000 max=65732 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] avg=32768.0000 max=33096 max_rank=5 min=32106 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=43479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[1866, 32574, 114, 65348, 183, 65465, 32673, 63921] avg=32768.0000 max=65465 max_rank=5 min=114 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=96705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=96705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[131, 80, 135, 65463, 65135, 65395, 33068, 32737] avg=32768.0000 max=65463 max_rank=3 min=80 min_rank=1 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[65179, 32778, 32821, 32661, 65732, 474, 32492, 7] avg=32768.0000 max=65732 max_rank=4 min=7 min_rank=7 imbalance_ratio=2.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] avg=32768.0000 max=33096 max_rank=3 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=95446 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] avg=32768.0000 max=33096 max_rank=0 min=32492 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32636 max_load_reduction_pct=0.496501 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] avg=32768.0000 max=33096 max_rank=3 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=95446 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=96705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[131, 80, 135, 65463, 65135, 65395, 33068, 32737] avg=32768.0000 max=65463 max_rank=3 min=80 min_rank=1 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[33096, 32778, 32821, 32661, 33096, 32557, 32492, 32643] avg=32768.0000 max=33096 max_rank=0 min=32492 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32636 max_load_reduction_pct=0.496501 moved_to_replicas=64719 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[1866, 32574, 114, 65348, 183, 65465, 32673, 63921] avg=32768.0000 max=65465 max_rank=5 min=114 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[1866, 32574, 114, 65348, 183, 65465, 32673, 63921] avg=32768.0000 max=65465 max_rank=5 min=114 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[35766, 268, 92608, 1945, 343, 624, 65401, 65189] avg=32768.0000 max=92608 max_rank=2 min=268 min_rank=1 imbalance_ratio=2.826172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[32787, 20876, 40815, 127, 32671, 65075, 32916, 36877] avg=32768.0000 max=65075 max_rank=5 min=127 min_rank=3 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[35766, 268, 92608, 1945, 343, 624, 65401, 65189] avg=32768.0000 max=92608 max_rank=2 min=268 min_rank=1 imbalance_ratio=2.826172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=96705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] avg=32768.0000 max=33096 max_rank=3 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=95446 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] avg=32768.0000 max=33096 max_rank=3 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=95446 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[1866, 32574, 114, 65348, 183, 65465, 32673, 63921] avg=32768.0000 max=65465 max_rank=5 min=114 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[32787, 20876, 40815, 127, 32671, 65075, 32916, 36877] avg=32768.0000 max=65075 max_rank=5 min=127 min_rank=3 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31032 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59512 max_load_reduction_pct=0.642623 moved_to_replicas=126580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] avg=32768.0000 max=33096 max_rank=5 min=32106 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=43479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31032 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59512 max_load_reduction_pct=0.642623 moved_to_replicas=126580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] avg=32768.0000 max=33096 max_rank=3 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=95446 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[1866, 32574, 114, 65348, 183, 65465, 32673, 63921] avg=32768.0000 max=65465 max_rank=5 min=114 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[32787, 32377, 33095, 32106, 32671, 33096, 32916, 33096] avg=32768.0000 max=33096 max_rank=5 min=32106 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=43479 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[35766, 268, 92608, 1945, 343, 624, 65401, 65189] avg=32768.0000 max=92608 max_rank=2 min=268 min_rank=1 imbalance_ratio=2.826172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[35766, 268, 92608, 1945, 343, 624, 65401, 65189] avg=32768.0000 max=92608 max_rank=2 min=268 min_rank=1 imbalance_ratio=2.826172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[32759, 57, 17, 77906, 218, 85625, 317, 65245] avg=32768.0000 max=85625 max_rank=5 min=17 min_rank=2 imbalance_ratio=2.613068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] avg=32768.0000 max=33096 max_rank=3 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=95446 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[32759, 57, 17, 77906, 218, 85625, 317, 65245] avg=32768.0000 max=85625 max_rank=5 min=17 min_rank=2 imbalance_ratio=2.613068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[131, 80, 135, 65463, 65135, 65395, 33068, 32737] avg=32768.0000 max=65463 max_rank=3 min=80 min_rank=1 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31032 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59512 max_load_reduction_pct=0.642623 moved_to_replicas=126580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31032 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59512 max_load_reduction_pct=0.642623 moved_to_replicas=126580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[35766, 268, 92608, 1945, 343, 624, 65401, 65189] avg=32768.0000 max=92608 max_rank=2 min=268 min_rank=1 imbalance_ratio=2.826172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[131, 80, 135, 65463, 65135, 65395, 33068, 32737] avg=32768.0000 max=65463 max_rank=3 min=80 min_rank=1 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=3 min=31566 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=52527 max_load_reduction_pct=0.613454 moved_to_replicas=129488 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=3 min=31566 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=52527 max_load_reduction_pct=0.613454 moved_to_replicas=129488 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=96705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[35766, 268, 92608, 1945, 343, 624, 65401, 65189] avg=32768.0000 max=92608 max_rank=2 min=268 min_rank=1 imbalance_ratio=2.826172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31032 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59512 max_load_reduction_pct=0.642623 moved_to_replicas=126580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[32759, 57, 17, 77906, 218, 85625, 317, 65245] avg=32768.0000 max=85625 max_rank=5 min=17 min_rank=2 imbalance_ratio=2.613068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[32430, 32447, 32174, 33096, 33096, 33096, 33068, 32737] avg=32768.0000 max=33096 max_rank=3 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=96705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[32759, 57, 17, 77906, 218, 85625, 317, 65245] avg=32768.0000 max=85625 max_rank=5 min=17 min_rank=2 imbalance_ratio=2.613068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[32907, 97143, 202, 1594, 732, 65300, 63923, 343] avg=32768.0000 max=97143 max_rank=1 min=202 min_rank=2 imbalance_ratio=2.964569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31032 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59512 max_load_reduction_pct=0.642623 moved_to_replicas=126580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[32907, 97143, 202, 1594, 732, 65300, 63923, 343] avg=32768.0000 max=97143 max_rank=1 min=202 min_rank=2 imbalance_ratio=2.964569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[1866, 32574, 114, 65348, 183, 65465, 32673, 63921] avg=32768.0000 max=65465 max_rank=5 min=114 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=3 min=31566 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=52527 max_load_reduction_pct=0.613454 moved_to_replicas=129488 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=3 min=31566 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=52527 max_load_reduction_pct=0.613454 moved_to_replicas=129488 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[32759, 57, 17, 77906, 218, 85625, 317, 65245] avg=32768.0000 max=85625 max_rank=5 min=17 min_rank=2 imbalance_ratio=2.613068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] avg=32768.0000 max=33096 max_rank=1 min=31816 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.659306 moved_to_replicas=127078 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[1866, 32574, 114, 65348, 183, 65465, 32673, 63921] avg=32768.0000 max=65465 max_rank=5 min=114 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] avg=32768.0000 max=33096 max_rank=1 min=31816 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.659306 moved_to_replicas=127078 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] avg=32768.0000 max=33096 max_rank=3 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=95446 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[32759, 57, 17, 77906, 218, 85625, 317, 65245] avg=32768.0000 max=85625 max_rank=5 min=17 min_rank=2 imbalance_ratio=2.613068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=3 min=31566 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=52527 max_load_reduction_pct=0.613454 moved_to_replicas=129488 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[32907, 97143, 202, 1594, 732, 65300, 63923, 343] avg=32768.0000 max=97143 max_rank=1 min=202 min_rank=2 imbalance_ratio=2.964569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[32691, 32574, 32483, 33096, 32435, 33096, 32673, 33096] avg=32768.0000 max=33096 max_rank=3 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=95446 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[32907, 97143, 202, 1594, 732, 65300, 63923, 343] avg=32768.0000 max=97143 max_rank=1 min=202 min_rank=2 imbalance_ratio=2.964569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[108, 32762, 65408, 32906, 32208, 35800, 29800, 33152] avg=32768.0000 max=65408 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=3 min=31566 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=52527 max_load_reduction_pct=0.613454 moved_to_replicas=129488 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[108, 32762, 65408, 32906, 32208, 35800, 29800, 33152] avg=32768.0000 max=65408 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] avg=32768.0000 max=33096 max_rank=1 min=31816 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.659306 moved_to_replicas=127078 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[35766, 268, 92608, 1945, 343, 624, 65401, 65189] avg=32768.0000 max=92608 max_rank=2 min=268 min_rank=1 imbalance_ratio=2.826172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] avg=32768.0000 max=33096 max_rank=1 min=31816 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.659306 moved_to_replicas=127078 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[32907, 97143, 202, 1594, 732, 65300, 63923, 343] avg=32768.0000 max=97143 max_rank=1 min=202 min_rank=2 imbalance_ratio=2.964569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] avg=32768.0000 max=33096 max_rank=2 min=32264 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32312 max_load_reduction_pct=0.494007 moved_to_replicas=35072 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[35766, 268, 92608, 1945, 343, 624, 65401, 65189] avg=32768.0000 max=92608 max_rank=2 min=268 min_rank=1 imbalance_ratio=2.826172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] avg=32768.0000 max=33096 max_rank=2 min=32264 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32312 max_load_reduction_pct=0.494007 moved_to_replicas=35072 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31032 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59512 max_load_reduction_pct=0.642623 moved_to_replicas=126580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[32907, 97143, 202, 1594, 732, 65300, 63923, 343] avg=32768.0000 max=97143 max_rank=1 min=202 min_rank=2 imbalance_ratio=2.964569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] avg=32768.0000 max=33096 max_rank=1 min=31816 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.659306 moved_to_replicas=127078 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[108, 32762, 65408, 32906, 32208, 35800, 29800, 33152] avg=32768.0000 max=65408 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[108, 32762, 65408, 32906, 32208, 35800, 29800, 33152] avg=32768.0000 max=65408 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[33096, 32703, 33096, 33096, 31032, 32929, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31032 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=59512 max_load_reduction_pct=0.642623 moved_to_replicas=126580 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32335, 32864, 65286, 32411, 1316, 32747] avg=32768.0000 max=65286 max_rank=4 min=1316 min_rank=6 imbalance_ratio=1.992371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] avg=32768.0000 max=33096 max_rank=1 min=31816 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.659306 moved_to_replicas=127078 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32335, 32864, 65286, 32411, 1316, 32747] avg=32768.0000 max=65286 max_rank=4 min=1316 min_rank=6 imbalance_ratio=1.992371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] avg=32768.0000 max=33096 max_rank=2 min=32264 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32312 max_load_reduction_pct=0.494007 moved_to_replicas=35072 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[32759, 57, 17, 77906, 218, 85625, 317, 65245] avg=32768.0000 max=85625 max_rank=5 min=17 min_rank=2 imbalance_ratio=2.613068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] avg=32768.0000 max=33096 max_rank=2 min=32264 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32312 max_load_reduction_pct=0.494007 moved_to_replicas=35072 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[108, 32762, 65408, 32906, 32208, 35800, 29800, 33152] avg=32768.0000 max=65408 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] avg=32768.0000 max=33096 max_rank=4 min=32411 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32190 max_load_reduction_pct=0.493061 moved_to_replicas=32190 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] avg=32768.0000 max=33096 max_rank=4 min=32411 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32190 max_load_reduction_pct=0.493061 moved_to_replicas=32190 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[32759, 57, 17, 77906, 218, 85625, 317, 65245] avg=32768.0000 max=85625 max_rank=5 min=17 min_rank=2 imbalance_ratio=2.613068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[108, 32762, 65408, 32906, 32208, 35800, 29800, 33152] avg=32768.0000 max=65408 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=3 min=31566 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=52527 max_load_reduction_pct=0.613454 moved_to_replicas=129488 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] avg=32768.0000 max=33096 max_rank=2 min=32264 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32312 max_load_reduction_pct=0.494007 moved_to_replicas=35072 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32335, 32864, 65286, 32411, 1316, 32747] avg=32768.0000 max=65286 max_rank=4 min=1316 min_rank=6 imbalance_ratio=1.992371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32335, 32864, 65286, 32411, 1316, 32747] avg=32768.0000 max=65286 max_rank=4 min=1316 min_rank=6 imbalance_ratio=1.992371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[32759, 31566, 32587, 33098, 32846, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=3 min=31566 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=52527 max_load_reduction_pct=0.613454 moved_to_replicas=129488 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=25 tokens_per_ep_rank=[9781, 32767, 38376, 65297, 1008, 65607, 16527, 32781] avg=32768.0000 max=65607 max_rank=5 min=1008 min_rank=4 imbalance_ratio=2.002167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] avg=32768.0000 max=33096 max_rank=2 min=32264 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32312 max_load_reduction_pct=0.494007 moved_to_replicas=35072 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=25 tokens_per_ep_rank=[9781, 32767, 38376, 65297, 1008, 65607, 16527, 32781] avg=32768.0000 max=65607 max_rank=5 min=1008 min_rank=4 imbalance_ratio=2.002167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] avg=32768.0000 max=33096 max_rank=4 min=32411 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32190 max_load_reduction_pct=0.493061 moved_to_replicas=32190 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] avg=32768.0000 max=33096 max_rank=4 min=32411 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32190 max_load_reduction_pct=0.493061 moved_to_replicas=32190 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[32907, 97143, 202, 1594, 732, 65300, 63923, 343] avg=32768.0000 max=97143 max_rank=1 min=202 min_rank=2 imbalance_ratio=2.964569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32335, 32864, 65286, 32411, 1316, 32747] avg=32768.0000 max=65286 max_rank=4 min=1316 min_rank=6 imbalance_ratio=1.992371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=25 tokens_per_ep_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] avg=32768.0000 max=33097 max_rank=0 min=31116 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32510 max_load_reduction_pct=0.495526 moved_to_replicas=69992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=25 tokens_per_ep_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] avg=32768.0000 max=33097 max_rank=0 min=31116 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32510 max_load_reduction_pct=0.495526 moved_to_replicas=69992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[32907, 97143, 202, 1594, 732, 65300, 63923, 343] avg=32768.0000 max=97143 max_rank=1 min=202 min_rank=2 imbalance_ratio=2.964569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32335, 32864, 65286, 32411, 1316, 32747] avg=32768.0000 max=65286 max_rank=4 min=1316 min_rank=6 imbalance_ratio=1.992371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] avg=32768.0000 max=33096 max_rank=1 min=31816 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.659306 moved_to_replicas=127078 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] avg=32768.0000 max=33096 max_rank=4 min=32411 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32190 max_load_reduction_pct=0.493061 moved_to_replicas=32190 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=25 tokens_per_ep_rank=[9781, 32767, 38376, 65297, 1008, 65607, 16527, 32781] avg=32768.0000 max=65607 max_rank=5 min=1008 min_rank=4 imbalance_ratio=2.002167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=25 tokens_per_ep_rank=[9781, 32767, 38376, 65297, 1008, 65607, 16527, 32781] avg=32768.0000 max=65607 max_rank=5 min=1008 min_rank=4 imbalance_ratio=2.002167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=26 tokens_per_ep_rank=[32122, 226, 33006, 32640, 97907, 461, 32764, 33018] avg=32768.0000 max=97907 max_rank=4 min=226 min_rank=1 imbalance_ratio=2.987885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[32907, 33096, 32776, 32421, 32936, 33096, 33096, 31816] avg=32768.0000 max=33096 max_rank=1 min=31816 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64047 max_load_reduction_pct=0.659306 moved_to_replicas=127078 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] avg=32768.0000 max=33096 max_rank=4 min=32411 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32190 max_load_reduction_pct=0.493061 moved_to_replicas=32190 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=26 tokens_per_ep_rank=[32122, 226, 33006, 32640, 97907, 461, 32764, 33018] avg=32768.0000 max=97907 max_rank=4 min=226 min_rank=1 imbalance_ratio=2.987885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=25 tokens_per_ep_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] avg=32768.0000 max=33097 max_rank=0 min=31116 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32510 max_load_reduction_pct=0.495526 moved_to_replicas=69992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=25 tokens_per_ep_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] avg=32768.0000 max=33097 max_rank=0 min=31116 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32510 max_load_reduction_pct=0.495526 moved_to_replicas=69992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64811 max_load_reduction_pct=0.661965 moved_to_replicas=64811 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[108, 32762, 65408, 32906, 32208, 35800, 29800, 33152] avg=32768.0000 max=65408 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=25 tokens_per_ep_rank=[9781, 32767, 38376, 65297, 1008, 65607, 16527, 32781] avg=32768.0000 max=65607 max_rank=5 min=1008 min_rank=4 imbalance_ratio=2.002167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64811 max_load_reduction_pct=0.661965 moved_to_replicas=64811 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=25 tokens_per_ep_rank=[9781, 32767, 38376, 65297, 1008, 65607, 16527, 32781] avg=32768.0000 max=65607 max_rank=5 min=1008 min_rank=4 imbalance_ratio=2.002167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[108, 32762, 65408, 32906, 32208, 35800, 29800, 33152] avg=32768.0000 max=65408 max_rank=2 min=108 min_rank=0 imbalance_ratio=1.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=26 tokens_per_ep_rank=[32122, 226, 33006, 32640, 97907, 461, 32764, 33018] avg=32768.0000 max=97907 max_rank=4 min=226 min_rank=1 imbalance_ratio=2.987885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=25 tokens_per_ep_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] avg=32768.0000 max=33097 max_rank=0 min=31116 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32510 max_load_reduction_pct=0.495526 moved_to_replicas=69992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] avg=32768.0000 max=33096 max_rank=2 min=32264 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32312 max_load_reduction_pct=0.494007 moved_to_replicas=35072 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=26 tokens_per_ep_rank=[32122, 226, 33006, 32640, 97907, 461, 32764, 33018] avg=32768.0000 max=97907 max_rank=4 min=226 min_rank=1 imbalance_ratio=2.987885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 43293, 268, 65203, 33036, 54980, 546] avg=32768.0000 max=65203 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=25 tokens_per_ep_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] avg=32768.0000 max=33097 max_rank=0 min=31116 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32510 max_load_reduction_pct=0.495526 moved_to_replicas=69992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 43293, 268, 65203, 33036, 54980, 546] avg=32768.0000 max=65203 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[32420, 32762, 33096, 32906, 32264, 33096, 32504, 33096] avg=32768.0000 max=33096 max_rank=2 min=32264 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32312 max_load_reduction_pct=0.494007 moved_to_replicas=35072 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64811 max_load_reduction_pct=0.661965 moved_to_replicas=64811 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64811 max_load_reduction_pct=0.661965 moved_to_replicas=64811 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] avg=32768.0000 max=33098 max_rank=2 min=31859 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32105 max_load_reduction_pct=0.492385 moved_to_replicas=64188 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=26 tokens_per_ep_rank=[32122, 226, 33006, 32640, 97907, 461, 32764, 33018] avg=32768.0000 max=97907 max_rank=4 min=226 min_rank=1 imbalance_ratio=2.987885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] avg=32768.0000 max=33098 max_rank=2 min=31859 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32105 max_load_reduction_pct=0.492385 moved_to_replicas=64188 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32335, 32864, 65286, 32411, 1316, 32747] avg=32768.0000 max=65286 max_rank=4 min=1316 min_rank=6 imbalance_ratio=1.992371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=26 tokens_per_ep_rank=[32122, 226, 33006, 32640, 97907, 461, 32764, 33018] avg=32768.0000 max=97907 max_rank=4 min=226 min_rank=1 imbalance_ratio=2.987885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32335, 32864, 65286, 32411, 1316, 32747] avg=32768.0000 max=65286 max_rank=4 min=1316 min_rank=6 imbalance_ratio=1.992371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 43293, 268, 65203, 33036, 54980, 546] avg=32768.0000 max=65203 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=28 tokens_per_ep_rank=[684, 32637, 733, 32580, 32755, 65208, 32393, 65154] avg=32768.0000 max=65208 max_rank=5 min=684 min_rank=0 imbalance_ratio=1.989990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 43293, 268, 65203, 33036, 54980, 546] avg=32768.0000 max=65203 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64811 max_load_reduction_pct=0.661965 moved_to_replicas=64811 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] avg=32768.0000 max=33096 max_rank=4 min=32411 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32190 max_load_reduction_pct=0.493061 moved_to_replicas=32190 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64811 max_load_reduction_pct=0.661965 moved_to_replicas=64811 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[32540, 32645, 32745, 32864, 33096, 32411, 33096, 32747] avg=32768.0000 max=33096 max_rank=4 min=32411 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32190 max_load_reduction_pct=0.493061 moved_to_replicas=32190 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=28 tokens_per_ep_rank=[684, 32637, 733, 32580, 32755, 65208, 32393, 65154] avg=32768.0000 max=65208 max_rank=5 min=684 min_rank=0 imbalance_ratio=1.989990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] avg=32768.0000 max=33098 max_rank=2 min=31859 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32105 max_load_reduction_pct=0.492385 moved_to_replicas=64188 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=28 tokens_per_ep_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] avg=32768.0000 max=33096 max_rank=5 min=32393 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32112 max_load_reduction_pct=0.492455 moved_to_replicas=64170 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] avg=32768.0000 max=33098 max_rank=2 min=31859 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32105 max_load_reduction_pct=0.492385 moved_to_replicas=64188 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 43293, 268, 65203, 33036, 54980, 546] avg=32768.0000 max=65203 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=28 tokens_per_ep_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] avg=32768.0000 max=33096 max_rank=5 min=32393 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32112 max_load_reduction_pct=0.492455 moved_to_replicas=64170 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 43293, 268, 65203, 33036, 54980, 546] avg=32768.0000 max=65203 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=25 tokens_per_ep_rank=[9781, 32767, 38376, 65297, 1008, 65607, 16527, 32781] avg=32768.0000 max=65607 max_rank=5 min=1008 min_rank=4 imbalance_ratio=2.002167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=25 tokens_per_ep_rank=[9781, 32767, 38376, 65297, 1008, 65607, 16527, 32781] avg=32768.0000 max=65607 max_rank=5 min=1008 min_rank=4 imbalance_ratio=2.002167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=28 tokens_per_ep_rank=[684, 32637, 733, 32580, 32755, 65208, 32393, 65154] avg=32768.0000 max=65208 max_rank=5 min=684 min_rank=0 imbalance_ratio=1.989990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 64837, 51715, 64728, 14446, 759, 726] avg=32768.0000 max=64837 max_rank=2 min=726 min_rank=7 imbalance_ratio=1.978668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=28 tokens_per_ep_rank=[684, 32637, 733, 32580, 32755, 65208, 32393, 65154] avg=32768.0000 max=65208 max_rank=5 min=684 min_rank=0 imbalance_ratio=1.989990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] avg=32768.0000 max=33098 max_rank=2 min=31859 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32105 max_load_reduction_pct=0.492385 moved_to_replicas=64188 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] avg=32768.0000 max=33098 max_rank=2 min=31859 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32105 max_load_reduction_pct=0.492385 moved_to_replicas=64188 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=25 tokens_per_ep_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] avg=32768.0000 max=33097 max_rank=0 min=31116 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32510 max_load_reduction_pct=0.495526 moved_to_replicas=69992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=25 tokens_per_ep_rank=[33097, 32767, 33095, 33096, 33096, 33096, 31116, 32781] avg=32768.0000 max=33097 max_rank=0 min=31116 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32510 max_load_reduction_pct=0.495526 moved_to_replicas=69992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=28 tokens_per_ep_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] avg=32768.0000 max=33096 max_rank=5 min=32393 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32112 max_load_reduction_pct=0.492455 moved_to_replicas=64170 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 64837, 51715, 64728, 14446, 759, 726] avg=32768.0000 max=64837 max_rank=2 min=726 min_rank=7 imbalance_ratio=1.978668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] avg=32768.0000 max=33098 max_rank=3 min=32008 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=31739 max_load_reduction_pct=0.489520 moved_to_replicas=81992 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=28 tokens_per_ep_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] avg=32768.0000 max=33096 max_rank=5 min=32393 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32112 max_load_reduction_pct=0.492455 moved_to_replicas=64170 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=28 tokens_per_ep_rank=[684, 32637, 733, 32580, 32755, 65208, 32393, 65154] avg=32768.0000 max=65208 max_rank=5 min=684 min_rank=0 imbalance_ratio=1.989990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=28 tokens_per_ep_rank=[684, 32637, 733, 32580, 32755, 65208, 32393, 65154] avg=32768.0000 max=65208 max_rank=5 min=684 min_rank=0 imbalance_ratio=1.989990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] avg=32768.0000 max=33098 max_rank=3 min=32008 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=31739 max_load_reduction_pct=0.489520 moved_to_replicas=81992 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=26 tokens_per_ep_rank=[32122, 226, 33006, 32640, 97907, 461, 32764, 33018] avg=32768.0000 max=97907 max_rank=4 min=226 min_rank=1 imbalance_ratio=2.987885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=26 tokens_per_ep_rank=[32122, 226, 33006, 32640, 97907, 461, 32764, 33018] avg=32768.0000 max=97907 max_rank=4 min=226 min_rank=1 imbalance_ratio=2.987885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 64837, 51715, 64728, 14446, 759, 726] avg=32768.0000 max=64837 max_rank=2 min=726 min_rank=7 imbalance_ratio=1.978668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=30 tokens_per_ep_rank=[29764, 32593, 32613, 65249, 65269, 32722, 3434, 500] avg=32768.0000 max=65269 max_rank=4 min=500 min_rank=7 imbalance_ratio=1.991852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=28 tokens_per_ep_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] avg=32768.0000 max=33096 max_rank=5 min=32393 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32112 max_load_reduction_pct=0.492455 moved_to_replicas=64170 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 64837, 51715, 64728, 14446, 759, 726] avg=32768.0000 max=64837 max_rank=2 min=726 min_rank=7 imbalance_ratio=1.978668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=28 tokens_per_ep_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] avg=32768.0000 max=33096 max_rank=5 min=32393 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32112 max_load_reduction_pct=0.492455 moved_to_replicas=64170 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64811 max_load_reduction_pct=0.661965 moved_to_replicas=64811 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] avg=32768.0000 max=33098 max_rank=3 min=32008 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=31739 max_load_reduction_pct=0.489520 moved_to_replicas=81992 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=30 tokens_per_ep_rank=[29764, 32593, 32613, 65249, 65269, 32722, 3434, 500] avg=32768.0000 max=65269 max_rank=4 min=500 min_rank=7 imbalance_ratio=1.991852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=26 tokens_per_ep_rank=[32122, 32794, 33006, 32640, 33096, 32704, 32764, 33018] avg=32768.0000 max=33096 max_rank=4 min=32122 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64811 max_load_reduction_pct=0.661965 moved_to_replicas=64811 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=30 tokens_per_ep_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] avg=32768.0000 max=33096 max_rank=3 min=32255 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32173 max_load_reduction_pct=0.492929 moved_to_replicas=64326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] avg=32768.0000 max=33098 max_rank=3 min=32008 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=31739 max_load_reduction_pct=0.489520 moved_to_replicas=81992 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 64837, 51715, 64728, 14446, 759, 726] avg=32768.0000 max=64837 max_rank=2 min=726 min_rank=7 imbalance_ratio=1.978668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 64837, 51715, 64728, 14446, 759, 726] avg=32768.0000 max=64837 max_rank=2 min=726 min_rank=7 imbalance_ratio=1.978668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=30 tokens_per_ep_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] avg=32768.0000 max=33096 max_rank=3 min=32255 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32173 max_load_reduction_pct=0.492929 moved_to_replicas=64326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 43293, 268, 65203, 33036, 54980, 546] avg=32768.0000 max=65203 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=30 tokens_per_ep_rank=[29764, 32593, 32613, 65249, 65269, 32722, 3434, 500] avg=32768.0000 max=65269 max_rank=4 min=500 min_rank=7 imbalance_ratio=1.991852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 43293, 268, 65203, 33036, 54980, 546] avg=32768.0000 max=65203 max_rank=4 min=268 min_rank=3 imbalance_ratio=1.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=31 tokens_per_ep_rank=[69353, 32831, 445, 125973, 47, 349, 46, 33100] avg=32768.0000 max=125973 max_rank=3 min=46 min_rank=6 imbalance_ratio=3.844391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] avg=32768.0000 max=33098 max_rank=3 min=32008 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=31739 max_load_reduction_pct=0.489520 moved_to_replicas=81992 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=30 tokens_per_ep_rank=[29764, 32593, 32613, 65249, 65269, 32722, 3434, 500] avg=32768.0000 max=65269 max_rank=4 min=500 min_rank=7 imbalance_ratio=1.991852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] avg=32768.0000 max=33098 max_rank=3 min=32008 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=31739 max_load_reduction_pct=0.489520 moved_to_replicas=81992 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] avg=32768.0000 max=33098 max_rank=2 min=31859 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32105 max_load_reduction_pct=0.492385 moved_to_replicas=64188 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=30 tokens_per_ep_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] avg=32768.0000 max=33096 max_rank=3 min=32255 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32173 max_load_reduction_pct=0.492929 moved_to_replicas=64326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=31 tokens_per_ep_rank=[69353, 32831, 445, 125973, 47, 349, 46, 33100] avg=32768.0000 max=125973 max_rank=3 min=46 min_rank=6 imbalance_ratio=3.844391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=27 tokens_per_ep_rank=[32959, 31859, 33098, 32375, 33096, 33036, 33095, 32626] avg=32768.0000 max=33098 max_rank=2 min=31859 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32105 max_load_reduction_pct=0.492385 moved_to_replicas=64188 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=92877 max_load_reduction_pct=0.737277 moved_to_replicas=129138 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=30 tokens_per_ep_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] avg=32768.0000 max=33096 max_rank=3 min=32255 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32173 max_load_reduction_pct=0.492929 moved_to_replicas=64326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=30 tokens_per_ep_rank=[29764, 32593, 32613, 65249, 65269, 32722, 3434, 500] avg=32768.0000 max=65269 max_rank=4 min=500 min_rank=7 imbalance_ratio=1.991852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=30 tokens_per_ep_rank=[29764, 32593, 32613, 65249, 65269, 32722, 3434, 500] avg=32768.0000 max=65269 max_rank=4 min=500 min_rank=7 imbalance_ratio=1.991852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=92877 max_load_reduction_pct=0.737277 moved_to_replicas=129138 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=28 tokens_per_ep_rank=[684, 32637, 733, 32580, 32755, 65208, 32393, 65154] avg=32768.0000 max=65208 max_rank=5 min=684 min_rank=0 imbalance_ratio=1.989990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=31 tokens_per_ep_rank=[69353, 32831, 445, 125973, 47, 349, 46, 33100] avg=32768.0000 max=125973 max_rank=3 min=46 min_rank=6 imbalance_ratio=3.844391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=28 tokens_per_ep_rank=[684, 32637, 733, 32580, 32755, 65208, 32393, 65154] avg=32768.0000 max=65208 max_rank=5 min=684 min_rank=0 imbalance_ratio=1.989990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=32 tokens_per_ep_rank=[418, 573, 190, 33443, 64924, 34020, 32694, 95882] avg=32768.0000 max=95882 max_rank=7 min=190 min_rank=2 imbalance_ratio=2.926086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=31 tokens_per_ep_rank=[69353, 32831, 445, 125973, 47, 349, 46, 33100] avg=32768.0000 max=125973 max_rank=3 min=46 min_rank=6 imbalance_ratio=3.844391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=30 tokens_per_ep_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] avg=32768.0000 max=33096 max_rank=3 min=32255 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32173 max_load_reduction_pct=0.492929 moved_to_replicas=64326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=30 tokens_per_ep_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] avg=32768.0000 max=33096 max_rank=3 min=32255 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32173 max_load_reduction_pct=0.492929 moved_to_replicas=64326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=28 tokens_per_ep_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] avg=32768.0000 max=33096 max_rank=5 min=32393 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32112 max_load_reduction_pct=0.492455 moved_to_replicas=64170 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=92877 max_load_reduction_pct=0.737277 moved_to_replicas=129138 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=32 tokens_per_ep_rank=[418, 573, 190, 33443, 64924, 34020, 32694, 95882] avg=32768.0000 max=95882 max_rank=7 min=190 min_rank=2 imbalance_ratio=2.926086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=28 tokens_per_ep_rank=[32796, 32637, 32791, 32580, 32755, 33096, 32393, 33096] avg=32768.0000 max=33096 max_rank=5 min=32393 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32112 max_load_reduction_pct=0.492455 moved_to_replicas=64170 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=32 tokens_per_ep_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] avg=32768.0000 max=33096 max_rank=3 min=31901 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62786 max_load_reduction_pct=0.654826 moved_to_replicas=95885 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=92877 max_load_reduction_pct=0.737277 moved_to_replicas=129138 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=31 tokens_per_ep_rank=[69353, 32831, 445, 125973, 47, 349, 46, 33100] avg=32768.0000 max=125973 max_rank=3 min=46 min_rank=6 imbalance_ratio=3.844391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=31 tokens_per_ep_rank=[69353, 32831, 445, 125973, 47, 349, 46, 33100] avg=32768.0000 max=125973 max_rank=3 min=46 min_rank=6 imbalance_ratio=3.844391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=32 tokens_per_ep_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] avg=32768.0000 max=33096 max_rank=3 min=31901 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62786 max_load_reduction_pct=0.654826 moved_to_replicas=95885 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 64837, 51715, 64728, 14446, 759, 726] avg=32768.0000 max=64837 max_rank=2 min=726 min_rank=7 imbalance_ratio=1.978668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=32 tokens_per_ep_rank=[418, 573, 190, 33443, 64924, 34020, 32694, 95882] avg=32768.0000 max=95882 max_rank=7 min=190 min_rank=2 imbalance_ratio=2.926086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=92877 max_load_reduction_pct=0.737277 moved_to_replicas=129138 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 64837, 51715, 64728, 14446, 759, 726] avg=32768.0000 max=64837 max_rank=2 min=726 min_rank=7 imbalance_ratio=1.978668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=33 tokens_per_ep_rank=[129756, 202, 34, 34989, 32610, 31392, 32805, 356] avg=32768.0000 max=129756 max_rank=0 min=34 min_rank=2 imbalance_ratio=3.959839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=92877 max_load_reduction_pct=0.737277 moved_to_replicas=129138 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=32 tokens_per_ep_rank=[418, 573, 190, 33443, 64924, 34020, 32694, 95882] avg=32768.0000 max=95882 max_rank=7 min=190 min_rank=2 imbalance_ratio=2.926086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] avg=32768.0000 max=33098 max_rank=3 min=32008 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=31739 max_load_reduction_pct=0.489520 moved_to_replicas=81992 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=32 tokens_per_ep_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] avg=32768.0000 max=33096 max_rank=3 min=31901 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62786 max_load_reduction_pct=0.654826 moved_to_replicas=95885 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=33 tokens_per_ep_rank=[129756, 202, 34, 34989, 32610, 31392, 32805, 356] avg=32768.0000 max=129756 max_rank=0 min=34 min_rank=2 imbalance_ratio=3.959839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=29 tokens_per_ep_rank=[32008, 32925, 33096, 33098, 33096, 33063, 32391, 32467] avg=32768.0000 max=33098 max_rank=3 min=32008 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=31739 max_load_reduction_pct=0.489520 moved_to_replicas=81992 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] avg=32768.0000 max=33096 max_rank=0 min=32162 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96660 max_load_reduction_pct=0.744937 moved_to_replicas=98553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=32 tokens_per_ep_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] avg=32768.0000 max=33096 max_rank=3 min=31901 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62786 max_load_reduction_pct=0.654826 moved_to_replicas=95885 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=32 tokens_per_ep_rank=[418, 573, 190, 33443, 64924, 34020, 32694, 95882] avg=32768.0000 max=95882 max_rank=7 min=190 min_rank=2 imbalance_ratio=2.926086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] avg=32768.0000 max=33096 max_rank=0 min=32162 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96660 max_load_reduction_pct=0.744937 moved_to_replicas=98553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=32 tokens_per_ep_rank=[418, 573, 190, 33443, 64924, 34020, 32694, 95882] avg=32768.0000 max=95882 max_rank=7 min=190 min_rank=2 imbalance_ratio=2.926086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=30 tokens_per_ep_rank=[29764, 32593, 32613, 65249, 65269, 32722, 3434, 500] avg=32768.0000 max=65269 max_rank=4 min=500 min_rank=7 imbalance_ratio=1.991852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=33 tokens_per_ep_rank=[129756, 202, 34, 34989, 32610, 31392, 32805, 356] avg=32768.0000 max=129756 max_rank=0 min=34 min_rank=2 imbalance_ratio=3.959839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=32 tokens_per_ep_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] avg=32768.0000 max=33096 max_rank=3 min=31901 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62786 max_load_reduction_pct=0.654826 moved_to_replicas=95885 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=30 tokens_per_ep_rank=[29764, 32593, 32613, 65249, 65269, 32722, 3434, 500] avg=32768.0000 max=65269 max_rank=4 min=500 min_rank=7 imbalance_ratio=1.991852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=34 tokens_per_ep_rank=[106249, 262, 493, 398, 285, 56505, 65286, 32666] avg=32768.0000 max=106249 max_rank=0 min=262 min_rank=1 imbalance_ratio=3.242462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=32 tokens_per_ep_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] avg=32768.0000 max=33096 max_rank=3 min=31901 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62786 max_load_reduction_pct=0.654826 moved_to_replicas=95885 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=33 tokens_per_ep_rank=[129756, 202, 34, 34989, 32610, 31392, 32805, 356] avg=32768.0000 max=129756 max_rank=0 min=34 min_rank=2 imbalance_ratio=3.959839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=30 tokens_per_ep_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] avg=32768.0000 max=33096 max_rank=3 min=32255 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32173 max_load_reduction_pct=0.492929 moved_to_replicas=64326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=34 tokens_per_ep_rank=[106249, 262, 493, 398, 285, 56505, 65286, 32666] avg=32768.0000 max=106249 max_rank=0 min=262 min_rank=1 imbalance_ratio=3.242462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] avg=32768.0000 max=33096 max_rank=0 min=32162 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96660 max_load_reduction_pct=0.744937 moved_to_replicas=98553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=30 tokens_per_ep_rank=[32255, 32593, 32613, 33096, 33096, 32722, 33096, 32673] avg=32768.0000 max=33096 max_rank=3 min=32255 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32173 max_load_reduction_pct=0.492929 moved_to_replicas=64326 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=34 tokens_per_ep_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] avg=32768.0000 max=33098 max_rank=0 min=31859 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=73151 max_load_reduction_pct=0.688486 moved_to_replicas=128752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] avg=32768.0000 max=33096 max_rank=0 min=32162 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96660 max_load_reduction_pct=0.744937 moved_to_replicas=98553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=34 tokens_per_ep_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] avg=32768.0000 max=33098 max_rank=0 min=31859 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=73151 max_load_reduction_pct=0.688486 moved_to_replicas=128752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=33 tokens_per_ep_rank=[129756, 202, 34, 34989, 32610, 31392, 32805, 356] avg=32768.0000 max=129756 max_rank=0 min=34 min_rank=2 imbalance_ratio=3.959839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=33 tokens_per_ep_rank=[129756, 202, 34, 34989, 32610, 31392, 32805, 356] avg=32768.0000 max=129756 max_rank=0 min=34 min_rank=2 imbalance_ratio=3.959839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=31 tokens_per_ep_rank=[69353, 32831, 445, 125973, 47, 349, 46, 33100] avg=32768.0000 max=125973 max_rank=3 min=46 min_rank=6 imbalance_ratio=3.844391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=34 tokens_per_ep_rank=[106249, 262, 493, 398, 285, 56505, 65286, 32666] avg=32768.0000 max=106249 max_rank=0 min=262 min_rank=1 imbalance_ratio=3.242462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] avg=32768.0000 max=33096 max_rank=0 min=32162 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96660 max_load_reduction_pct=0.744937 moved_to_replicas=98553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=35 tokens_per_ep_rank=[1306, 43572, 32621, 2700, 64232, 157, 52371, 65185] avg=32768.0000 max=65185 max_rank=7 min=157 min_rank=5 imbalance_ratio=1.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=31 tokens_per_ep_rank=[69353, 32831, 445, 125973, 47, 349, 46, 33100] avg=32768.0000 max=125973 max_rank=3 min=46 min_rank=6 imbalance_ratio=3.844391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=34 tokens_per_ep_rank=[106249, 262, 493, 398, 285, 56505, 65286, 32666] avg=32768.0000 max=106249 max_rank=0 min=262 min_rank=1 imbalance_ratio=3.242462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] avg=32768.0000 max=33096 max_rank=0 min=32162 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96660 max_load_reduction_pct=0.744937 moved_to_replicas=98553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=92877 max_load_reduction_pct=0.737277 moved_to_replicas=129138 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=34 tokens_per_ep_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] avg=32768.0000 max=33098 max_rank=0 min=31859 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=73151 max_load_reduction_pct=0.688486 moved_to_replicas=128752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=35 tokens_per_ep_rank=[1306, 43572, 32621, 2700, 64232, 157, 52371, 65185] avg=32768.0000 max=65185 max_rank=7 min=157 min_rank=5 imbalance_ratio=1.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=35 tokens_per_ep_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] avg=32768.0000 max=33096 max_rank=4 min=32246 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32089 max_load_reduction_pct=0.492276 moved_to_replicas=92976 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=31 tokens_per_ep_rank=[33096, 32831, 32937, 33094, 32548, 31944, 32598, 33096] avg=32768.0000 max=33096 max_rank=0 min=31944 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=92877 max_load_reduction_pct=0.737277 moved_to_replicas=129138 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=34 tokens_per_ep_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] avg=32768.0000 max=33098 max_rank=0 min=31859 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=73151 max_load_reduction_pct=0.688486 moved_to_replicas=128752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=35 tokens_per_ep_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] avg=32768.0000 max=33096 max_rank=4 min=32246 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32089 max_load_reduction_pct=0.492276 moved_to_replicas=92976 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=34 tokens_per_ep_rank=[106249, 262, 493, 398, 285, 56505, 65286, 32666] avg=32768.0000 max=106249 max_rank=0 min=262 min_rank=1 imbalance_ratio=3.242462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=34 tokens_per_ep_rank=[106249, 262, 493, 398, 285, 56505, 65286, 32666] avg=32768.0000 max=106249 max_rank=0 min=262 min_rank=1 imbalance_ratio=3.242462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=32 tokens_per_ep_rank=[418, 573, 190, 33443, 64924, 34020, 32694, 95882] avg=32768.0000 max=95882 max_rank=7 min=190 min_rank=2 imbalance_ratio=2.926086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=35 tokens_per_ep_rank=[1306, 43572, 32621, 2700, 64232, 157, 52371, 65185] avg=32768.0000 max=65185 max_rank=7 min=157 min_rank=5 imbalance_ratio=1.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=34 tokens_per_ep_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] avg=32768.0000 max=33098 max_rank=0 min=31859 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=73151 max_load_reduction_pct=0.688486 moved_to_replicas=128752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=36 tokens_per_ep_rank=[91528, 34207, 402, 5217, 32720, 32766, 32580, 32724] avg=32768.0000 max=91528 max_rank=0 min=402 min_rank=2 imbalance_ratio=2.793213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=34 tokens_per_ep_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] avg=32768.0000 max=33098 max_rank=0 min=31859 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=73151 max_load_reduction_pct=0.688486 moved_to_replicas=128752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=32 tokens_per_ep_rank=[418, 573, 190, 33443, 64924, 34020, 32694, 95882] avg=32768.0000 max=95882 max_rank=7 min=190 min_rank=2 imbalance_ratio=2.926086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=35 tokens_per_ep_rank=[1306, 43572, 32621, 2700, 64232, 157, 52371, 65185] avg=32768.0000 max=65185 max_rank=7 min=157 min_rank=5 imbalance_ratio=1.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=32 tokens_per_ep_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] avg=32768.0000 max=33096 max_rank=3 min=31901 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62786 max_load_reduction_pct=0.654826 moved_to_replicas=95885 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=35 tokens_per_ep_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] avg=32768.0000 max=33096 max_rank=4 min=32246 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32089 max_load_reduction_pct=0.492276 moved_to_replicas=92976 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=36 tokens_per_ep_rank=[91528, 34207, 402, 5217, 32720, 32766, 32580, 32724] avg=32768.0000 max=91528 max_rank=0 min=402 min_rank=2 imbalance_ratio=2.793213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] avg=32768.0000 max=33096 max_rank=0 min=32288 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58432 max_load_reduction_pct=0.638406 moved_to_replicas=59543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=32 tokens_per_ep_rank=[31901, 32401, 32764, 33096, 33096, 33096, 32694, 33096] avg=32768.0000 max=33096 max_rank=3 min=31901 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62786 max_load_reduction_pct=0.654826 moved_to_replicas=95885 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=35 tokens_per_ep_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] avg=32768.0000 max=33096 max_rank=4 min=32246 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32089 max_load_reduction_pct=0.492276 moved_to_replicas=92976 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=35 tokens_per_ep_rank=[1306, 43572, 32621, 2700, 64232, 157, 52371, 65185] avg=32768.0000 max=65185 max_rank=7 min=157 min_rank=5 imbalance_ratio=1.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] avg=32768.0000 max=33096 max_rank=0 min=32288 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58432 max_load_reduction_pct=0.638406 moved_to_replicas=59543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=35 tokens_per_ep_rank=[1306, 43572, 32621, 2700, 64232, 157, 52371, 65185] avg=32768.0000 max=65185 max_rank=7 min=157 min_rank=5 imbalance_ratio=1.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=33 tokens_per_ep_rank=[129756, 202, 34, 34989, 32610, 31392, 32805, 356] avg=32768.0000 max=129756 max_rank=0 min=34 min_rank=2 imbalance_ratio=3.959839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=36 tokens_per_ep_rank=[91528, 34207, 402, 5217, 32720, 32766, 32580, 32724] avg=32768.0000 max=91528 max_rank=0 min=402 min_rank=2 imbalance_ratio=2.793213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=35 tokens_per_ep_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] avg=32768.0000 max=33096 max_rank=4 min=32246 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32089 max_load_reduction_pct=0.492276 moved_to_replicas=92976 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=37 tokens_per_ep_rank=[546, 64654, 33512, 32992, 32814, 101, 32450, 65075] avg=32768.0000 max=65075 max_rank=7 min=101 min_rank=5 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=35 tokens_per_ep_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] avg=32768.0000 max=33096 max_rank=4 min=32246 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32089 max_load_reduction_pct=0.492276 moved_to_replicas=92976 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=36 tokens_per_ep_rank=[91528, 34207, 402, 5217, 32720, 32766, 32580, 32724] avg=32768.0000 max=91528 max_rank=0 min=402 min_rank=2 imbalance_ratio=2.793213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=33 tokens_per_ep_rank=[129756, 202, 34, 34989, 32610, 31392, 32805, 356] avg=32768.0000 max=129756 max_rank=0 min=34 min_rank=2 imbalance_ratio=3.959839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=37 tokens_per_ep_rank=[546, 64654, 33512, 32992, 32814, 101, 32450, 65075] avg=32768.0000 max=65075 max_rank=7 min=101 min_rank=5 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] avg=32768.0000 max=33096 max_rank=0 min=32288 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58432 max_load_reduction_pct=0.638406 moved_to_replicas=59543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] avg=32768.0000 max=33096 max_rank=0 min=32162 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96660 max_load_reduction_pct=0.744937 moved_to_replicas=98553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=63953 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=36 tokens_per_ep_rank=[91528, 34207, 402, 5217, 32720, 32766, 32580, 32724] avg=32768.0000 max=91528 max_rank=0 min=402 min_rank=2 imbalance_ratio=2.793213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] avg=32768.0000 max=33096 max_rank=0 min=32288 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58432 max_load_reduction_pct=0.638406 moved_to_replicas=59543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=33 tokens_per_ep_rank=[33096, 32711, 32568, 33096, 32610, 33096, 32805, 32162] avg=32768.0000 max=33096 max_rank=0 min=32162 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96660 max_load_reduction_pct=0.744937 moved_to_replicas=98553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=63953 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=36 tokens_per_ep_rank=[91528, 34207, 402, 5217, 32720, 32766, 32580, 32724] avg=32768.0000 max=91528 max_rank=0 min=402 min_rank=2 imbalance_ratio=2.793213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=37 tokens_per_ep_rank=[546, 64654, 33512, 32992, 32814, 101, 32450, 65075] avg=32768.0000 max=65075 max_rank=7 min=101 min_rank=5 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=34 tokens_per_ep_rank=[106249, 262, 493, 398, 285, 56505, 65286, 32666] avg=32768.0000 max=106249 max_rank=0 min=262 min_rank=1 imbalance_ratio=3.242462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] avg=32768.0000 max=33096 max_rank=0 min=32288 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58432 max_load_reduction_pct=0.638406 moved_to_replicas=59543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=38 tokens_per_ep_rank=[3784, 32984, 32622, 61845, 258, 32504, 32721, 65426] avg=32768.0000 max=65426 max_rank=7 min=258 min_rank=4 imbalance_ratio=1.996643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] avg=32768.0000 max=33096 max_rank=0 min=32288 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58432 max_load_reduction_pct=0.638406 moved_to_replicas=59543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=37 tokens_per_ep_rank=[546, 64654, 33512, 32992, 32814, 101, 32450, 65075] avg=32768.0000 max=65075 max_rank=7 min=101 min_rank=5 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=34 tokens_per_ep_rank=[106249, 262, 493, 398, 285, 56505, 65286, 32666] avg=32768.0000 max=106249 max_rank=0 min=262 min_rank=1 imbalance_ratio=3.242462 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=38 tokens_per_ep_rank=[3784, 32984, 32622, 61845, 258, 32504, 32721, 65426] avg=32768.0000 max=65426 max_rank=7 min=258 min_rank=4 imbalance_ratio=1.996643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=63953 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=34 tokens_per_ep_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] avg=32768.0000 max=33098 max_rank=0 min=31859 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=73151 max_load_reduction_pct=0.688486 moved_to_replicas=128752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] avg=32768.0000 max=33096 max_rank=3 min=32504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32330 max_load_reduction_pct=0.494146 moved_to_replicas=61079 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=63953 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=34 tokens_per_ep_rank=[33098, 32833, 32683, 31859, 32812, 33097, 33096, 32666] avg=32768.0000 max=33098 max_rank=0 min=31859 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=73151 max_load_reduction_pct=0.688486 moved_to_replicas=128752 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=37 tokens_per_ep_rank=[546, 64654, 33512, 32992, 32814, 101, 32450, 65075] avg=32768.0000 max=65075 max_rank=7 min=101 min_rank=5 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] avg=32768.0000 max=33096 max_rank=3 min=32504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32330 max_load_reduction_pct=0.494146 moved_to_replicas=61079 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=37 tokens_per_ep_rank=[546, 64654, 33512, 32992, 32814, 101, 32450, 65075] avg=32768.0000 max=65075 max_rank=7 min=101 min_rank=5 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=38 tokens_per_ep_rank=[3784, 32984, 32622, 61845, 258, 32504, 32721, 65426] avg=32768.0000 max=65426 max_rank=7 min=258 min_rank=4 imbalance_ratio=1.996643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=39 tokens_per_ep_rank=[97579, 164, 32716, 665, 32878, 563, 62065, 35514] avg=32768.0000 max=97579 max_rank=0 min=164 min_rank=1 imbalance_ratio=2.977875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=35 tokens_per_ep_rank=[1306, 43572, 32621, 2700, 64232, 157, 52371, 65185] avg=32768.0000 max=65185 max_rank=7 min=157 min_rank=5 imbalance_ratio=1.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=63953 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=63953 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=38 tokens_per_ep_rank=[3784, 32984, 32622, 61845, 258, 32504, 32721, 65426] avg=32768.0000 max=65426 max_rank=7 min=258 min_rank=4 imbalance_ratio=1.996643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=35 tokens_per_ep_rank=[1306, 43572, 32621, 2700, 64232, 157, 52371, 65185] avg=32768.0000 max=65185 max_rank=7 min=157 min_rank=5 imbalance_ratio=1.989288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=39 tokens_per_ep_rank=[97579, 164, 32716, 665, 32878, 563, 62065, 35514] avg=32768.0000 max=97579 max_rank=0 min=164 min_rank=1 imbalance_ratio=2.977875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] avg=32768.0000 max=33096 max_rank=3 min=32504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32330 max_load_reduction_pct=0.494146 moved_to_replicas=61079 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32052 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64483 max_load_reduction_pct=0.660829 moved_to_replicas=95870 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=35 tokens_per_ep_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] avg=32768.0000 max=33096 max_rank=4 min=32246 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32089 max_load_reduction_pct=0.492276 moved_to_replicas=92976 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] avg=32768.0000 max=33096 max_rank=3 min=32504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32330 max_load_reduction_pct=0.494146 moved_to_replicas=61079 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=35 tokens_per_ep_rank=[32442, 33095, 32621, 32453, 33096, 32246, 33095, 33096] avg=32768.0000 max=33096 max_rank=4 min=32246 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32089 max_load_reduction_pct=0.492276 moved_to_replicas=92976 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=38 tokens_per_ep_rank=[3784, 32984, 32622, 61845, 258, 32504, 32721, 65426] avg=32768.0000 max=65426 max_rank=7 min=258 min_rank=4 imbalance_ratio=1.996643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=38 tokens_per_ep_rank=[3784, 32984, 32622, 61845, 258, 32504, 32721, 65426] avg=32768.0000 max=65426 max_rank=7 min=258 min_rank=4 imbalance_ratio=1.996643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32052 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64483 max_load_reduction_pct=0.660829 moved_to_replicas=95870 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=39 tokens_per_ep_rank=[97579, 164, 32716, 665, 32878, 563, 62065, 35514] avg=32768.0000 max=97579 max_rank=0 min=164 min_rank=1 imbalance_ratio=2.977875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1125, 33106, 65339, 492, 65112, 65156, 225] avg=32768.0000 max=65339 max_rank=3 min=225 min_rank=7 imbalance_ratio=1.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=36 tokens_per_ep_rank=[91528, 34207, 402, 5217, 32720, 32766, 32580, 32724] avg=32768.0000 max=91528 max_rank=0 min=402 min_rank=2 imbalance_ratio=2.793213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=39 tokens_per_ep_rank=[97579, 164, 32716, 665, 32878, 563, 62065, 35514] avg=32768.0000 max=97579 max_rank=0 min=164 min_rank=1 imbalance_ratio=2.977875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] avg=32768.0000 max=33096 max_rank=3 min=32504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32330 max_load_reduction_pct=0.494146 moved_to_replicas=61079 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] avg=32768.0000 max=33096 max_rank=3 min=32504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32330 max_load_reduction_pct=0.494146 moved_to_replicas=61079 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=36 tokens_per_ep_rank=[91528, 34207, 402, 5217, 32720, 32766, 32580, 32724] avg=32768.0000 max=91528 max_rank=0 min=402 min_rank=2 imbalance_ratio=2.793213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32052 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64483 max_load_reduction_pct=0.660829 moved_to_replicas=95870 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1125, 33106, 65339, 492, 65112, 65156, 225] avg=32768.0000 max=65339 max_rank=3 min=225 min_rank=7 imbalance_ratio=1.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=40 tokens_per_ep_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] avg=32768.0000 max=33096 max_rank=1 min=31644 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32243 max_load_reduction_pct=0.493473 moved_to_replicas=96329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] avg=32768.0000 max=33096 max_rank=0 min=32288 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58432 max_load_reduction_pct=0.638406 moved_to_replicas=59543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32052 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64483 max_load_reduction_pct=0.660829 moved_to_replicas=95870 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=36 tokens_per_ep_rank=[33096, 33096, 32874, 32288, 32720, 32766, 32580, 32724] avg=32768.0000 max=33096 max_rank=0 min=32288 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=58432 max_load_reduction_pct=0.638406 moved_to_replicas=59543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=40 tokens_per_ep_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] avg=32768.0000 max=33096 max_rank=1 min=31644 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32243 max_load_reduction_pct=0.493473 moved_to_replicas=96329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=39 tokens_per_ep_rank=[97579, 164, 32716, 665, 32878, 563, 62065, 35514] avg=32768.0000 max=97579 max_rank=0 min=164 min_rank=1 imbalance_ratio=2.977875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=39 tokens_per_ep_rank=[97579, 164, 32716, 665, 32878, 563, 62065, 35514] avg=32768.0000 max=97579 max_rank=0 min=164 min_rank=1 imbalance_ratio=2.977875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1125, 33106, 65339, 492, 65112, 65156, 225] avg=32768.0000 max=65339 max_rank=3 min=225 min_rank=7 imbalance_ratio=1.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=41 tokens_per_ep_rank=[410, 65248, 401, 32633, 65043, 32245, 787, 65377] avg=32768.0000 max=65377 max_rank=7 min=401 min_rank=2 imbalance_ratio=1.995148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=37 tokens_per_ep_rank=[546, 64654, 33512, 32992, 32814, 101, 32450, 65075] avg=32768.0000 max=65075 max_rank=7 min=101 min_rank=5 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1125, 33106, 65339, 492, 65112, 65156, 225] avg=32768.0000 max=65339 max_rank=3 min=225 min_rank=7 imbalance_ratio=1.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32052 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64483 max_load_reduction_pct=0.660829 moved_to_replicas=95870 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32052 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64483 max_load_reduction_pct=0.660829 moved_to_replicas=95870 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=40 tokens_per_ep_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] avg=32768.0000 max=33096 max_rank=1 min=31644 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32243 max_load_reduction_pct=0.493473 moved_to_replicas=96329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=41 tokens_per_ep_rank=[410, 65248, 401, 32633, 65043, 32245, 787, 65377] avg=32768.0000 max=65377 max_rank=7 min=401 min_rank=2 imbalance_ratio=1.995148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=37 tokens_per_ep_rank=[546, 64654, 33512, 32992, 32814, 101, 32450, 65075] avg=32768.0000 max=65075 max_rank=7 min=101 min_rank=5 imbalance_ratio=1.985931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32281 max_load_reduction_pct=0.493767 moved_to_replicas=96380 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=63953 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=40 tokens_per_ep_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] avg=32768.0000 max=33096 max_rank=1 min=31644 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32243 max_load_reduction_pct=0.493473 moved_to_replicas=96329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32281 max_load_reduction_pct=0.493767 moved_to_replicas=96380 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=37 tokens_per_ep_rank=[32104, 33096, 33096, 32992, 32814, 32496, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31979 max_load_reduction_pct=0.491418 moved_to_replicas=63953 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1125, 33106, 65339, 492, 65112, 65156, 225] avg=32768.0000 max=65339 max_rank=3 min=225 min_rank=7 imbalance_ratio=1.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1125, 33106, 65339, 492, 65112, 65156, 225] avg=32768.0000 max=65339 max_rank=3 min=225 min_rank=7 imbalance_ratio=1.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=41 tokens_per_ep_rank=[410, 65248, 401, 32633, 65043, 32245, 787, 65377] avg=32768.0000 max=65377 max_rank=7 min=401 min_rank=2 imbalance_ratio=1.995148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=42 tokens_per_ep_rank=[65391, 30929, 377, 129834, 259, 272, 2670, 32412] avg=32768.0000 max=129834 max_rank=3 min=259 min_rank=4 imbalance_ratio=3.962219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=38 tokens_per_ep_rank=[3784, 32984, 32622, 61845, 258, 32504, 32721, 65426] avg=32768.0000 max=65426 max_rank=7 min=258 min_rank=4 imbalance_ratio=1.996643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=41 tokens_per_ep_rank=[410, 65248, 401, 32633, 65043, 32245, 787, 65377] avg=32768.0000 max=65377 max_rank=7 min=401 min_rank=2 imbalance_ratio=1.995148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=40 tokens_per_ep_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] avg=32768.0000 max=33096 max_rank=1 min=31644 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32243 max_load_reduction_pct=0.493473 moved_to_replicas=96329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=40 tokens_per_ep_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] avg=32768.0000 max=33096 max_rank=1 min=31644 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32243 max_load_reduction_pct=0.493473 moved_to_replicas=96329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32281 max_load_reduction_pct=0.493767 moved_to_replicas=96380 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=42 tokens_per_ep_rank=[65391, 30929, 377, 129834, 259, 272, 2670, 32412] avg=32768.0000 max=129834 max_rank=3 min=259 min_rank=4 imbalance_ratio=3.962219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=38 tokens_per_ep_rank=[3784, 32984, 32622, 61845, 258, 32504, 32721, 65426] avg=32768.0000 max=65426 max_rank=7 min=258 min_rank=4 imbalance_ratio=1.996643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] avg=32768.0000 max=33096 max_rank=0 min=32084 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96738 max_load_reduction_pct=0.745090 moved_to_replicas=129033 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] avg=32768.0000 max=33096 max_rank=3 min=32504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32330 max_load_reduction_pct=0.494146 moved_to_replicas=61079 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32281 max_load_reduction_pct=0.493767 moved_to_replicas=96380 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] avg=32768.0000 max=33096 max_rank=0 min=32084 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96738 max_load_reduction_pct=0.745090 moved_to_replicas=129033 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=38 tokens_per_ep_rank=[32533, 32984, 32622, 33096, 32588, 32504, 32721, 33096] avg=32768.0000 max=33096 max_rank=3 min=32504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32330 max_load_reduction_pct=0.494146 moved_to_replicas=61079 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=41 tokens_per_ep_rank=[410, 65248, 401, 32633, 65043, 32245, 787, 65377] avg=32768.0000 max=65377 max_rank=7 min=401 min_rank=2 imbalance_ratio=1.995148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=41 tokens_per_ep_rank=[410, 65248, 401, 32633, 65043, 32245, 787, 65377] avg=32768.0000 max=65377 max_rank=7 min=401 min_rank=2 imbalance_ratio=1.995148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=42 tokens_per_ep_rank=[65391, 30929, 377, 129834, 259, 272, 2670, 32412] avg=32768.0000 max=129834 max_rank=3 min=259 min_rank=4 imbalance_ratio=3.962219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 29973, 260, 38922, 32582, 63014, 31407] avg=32768.0000 max=63014 max_rank=6 min=260 min_rank=3 imbalance_ratio=1.923035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32281 max_load_reduction_pct=0.493767 moved_to_replicas=96380 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32281 max_load_reduction_pct=0.493767 moved_to_replicas=96380 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=39 tokens_per_ep_rank=[97579, 164, 32716, 665, 32878, 563, 62065, 35514] avg=32768.0000 max=97579 max_rank=0 min=164 min_rank=1 imbalance_ratio=2.977875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] avg=32768.0000 max=33096 max_rank=0 min=32084 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96738 max_load_reduction_pct=0.745090 moved_to_replicas=129033 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=42 tokens_per_ep_rank=[65391, 30929, 377, 129834, 259, 272, 2670, 32412] avg=32768.0000 max=129834 max_rank=3 min=259 min_rank=4 imbalance_ratio=3.962219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 29973, 260, 38922, 32582, 63014, 31407] avg=32768.0000 max=63014 max_rank=6 min=260 min_rank=3 imbalance_ratio=1.923035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=39 tokens_per_ep_rank=[97579, 164, 32716, 665, 32878, 563, 62065, 35514] avg=32768.0000 max=97579 max_rank=0 min=164 min_rank=1 imbalance_ratio=2.977875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] avg=32768.0000 max=33096 max_rank=2 min=31407 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29918 max_load_reduction_pct=0.474783 moved_to_replicas=35744 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32052 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64483 max_load_reduction_pct=0.660829 moved_to_replicas=95870 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] avg=32768.0000 max=33096 max_rank=0 min=32084 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96738 max_load_reduction_pct=0.745090 moved_to_replicas=129033 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=42 tokens_per_ep_rank=[65391, 30929, 377, 129834, 259, 272, 2670, 32412] avg=32768.0000 max=129834 max_rank=3 min=259 min_rank=4 imbalance_ratio=3.962219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] avg=32768.0000 max=33096 max_rank=2 min=31407 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29918 max_load_reduction_pct=0.474783 moved_to_replicas=35744 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=39 tokens_per_ep_rank=[33096, 32608, 32716, 32052, 32878, 32602, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32052 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64483 max_load_reduction_pct=0.660829 moved_to_replicas=95870 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=42 tokens_per_ep_rank=[65391, 30929, 377, 129834, 259, 272, 2670, 32412] avg=32768.0000 max=129834 max_rank=3 min=259 min_rank=4 imbalance_ratio=3.962219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 29973, 260, 38922, 32582, 63014, 31407] avg=32768.0000 max=63014 max_rank=6 min=260 min_rank=3 imbalance_ratio=1.923035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=44 tokens_per_ep_rank=[39576, 89874, 159, 64868, 391, 66597, 544, 135] avg=32768.0000 max=89874 max_rank=1 min=135 min_rank=7 imbalance_ratio=2.742737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] avg=32768.0000 max=33096 max_rank=0 min=32084 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96738 max_load_reduction_pct=0.745090 moved_to_replicas=129033 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] avg=32768.0000 max=33096 max_rank=0 min=32084 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96738 max_load_reduction_pct=0.745090 moved_to_replicas=129033 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] avg=32768.0000 max=33096 max_rank=2 min=31407 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29918 max_load_reduction_pct=0.474783 moved_to_replicas=35744 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1125, 33106, 65339, 492, 65112, 65156, 225] avg=32768.0000 max=65339 max_rank=3 min=225 min_rank=7 imbalance_ratio=1.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 29973, 260, 38922, 32582, 63014, 31407] avg=32768.0000 max=63014 max_rank=6 min=260 min_rank=3 imbalance_ratio=1.923035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=44 tokens_per_ep_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] avg=32768.0000 max=33096 max_rank=0 min=31228 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56778 max_load_reduction_pct=0.631751 moved_to_replicas=128531 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=40 tokens_per_ep_rank=[31589, 1125, 33106, 65339, 492, 65112, 65156, 225] avg=32768.0000 max=65339 max_rank=3 min=225 min_rank=7 imbalance_ratio=1.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=44 tokens_per_ep_rank=[39576, 89874, 159, 64868, 391, 66597, 544, 135] avg=32768.0000 max=89874 max_rank=1 min=135 min_rank=7 imbalance_ratio=2.742737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 29973, 260, 38922, 32582, 63014, 31407] avg=32768.0000 max=63014 max_rank=6 min=260 min_rank=3 imbalance_ratio=1.923035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=40 tokens_per_ep_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] avg=32768.0000 max=33096 max_rank=1 min=31644 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32243 max_load_reduction_pct=0.493473 moved_to_replicas=96329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] avg=32768.0000 max=33096 max_rank=2 min=31407 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29918 max_load_reduction_pct=0.474783 moved_to_replicas=35744 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=40 tokens_per_ep_rank=[31644, 33096, 33096, 33096, 32552, 33096, 33096, 32468] avg=32768.0000 max=33096 max_rank=1 min=31644 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32243 max_load_reduction_pct=0.493473 moved_to_replicas=96329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=44 tokens_per_ep_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] avg=32768.0000 max=33096 max_rank=0 min=31228 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56778 max_load_reduction_pct=0.631751 moved_to_replicas=128531 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 29973, 260, 38922, 32582, 63014, 31407] avg=32768.0000 max=63014 max_rank=6 min=260 min_rank=3 imbalance_ratio=1.923035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=44 tokens_per_ep_rank=[39576, 89874, 159, 64868, 391, 66597, 544, 135] avg=32768.0000 max=89874 max_rank=1 min=135 min_rank=7 imbalance_ratio=2.742737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=45 tokens_per_ep_rank=[32115, 64846, 32421, 236, 65447, 32556, 33486, 1037] avg=32768.0000 max=65447 max_rank=4 min=236 min_rank=3 imbalance_ratio=1.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] avg=32768.0000 max=33096 max_rank=2 min=31407 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29918 max_load_reduction_pct=0.474783 moved_to_replicas=35744 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] avg=32768.0000 max=33096 max_rank=2 min=31407 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29918 max_load_reduction_pct=0.474783 moved_to_replicas=35744 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=44 tokens_per_ep_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] avg=32768.0000 max=33096 max_rank=0 min=31228 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56778 max_load_reduction_pct=0.631751 moved_to_replicas=128531 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=45 tokens_per_ep_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32351 max_load_reduction_pct=0.494308 moved_to_replicas=64491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=44 tokens_per_ep_rank=[39576, 89874, 159, 64868, 391, 66597, 544, 135] avg=32768.0000 max=89874 max_rank=1 min=135 min_rank=7 imbalance_ratio=2.742737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=41 tokens_per_ep_rank=[410, 65248, 401, 32633, 65043, 32245, 787, 65377] avg=32768.0000 max=65377 max_rank=7 min=401 min_rank=2 imbalance_ratio=1.995148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=45 tokens_per_ep_rank=[32115, 64846, 32421, 236, 65447, 32556, 33486, 1037] avg=32768.0000 max=65447 max_rank=4 min=236 min_rank=3 imbalance_ratio=1.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=41 tokens_per_ep_rank=[410, 65248, 401, 32633, 65043, 32245, 787, 65377] avg=32768.0000 max=65377 max_rank=7 min=401 min_rank=2 imbalance_ratio=1.995148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=44 tokens_per_ep_rank=[39576, 89874, 159, 64868, 391, 66597, 544, 135] avg=32768.0000 max=89874 max_rank=1 min=135 min_rank=7 imbalance_ratio=2.742737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=44 tokens_per_ep_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] avg=32768.0000 max=33096 max_rank=0 min=31228 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56778 max_load_reduction_pct=0.631751 moved_to_replicas=128531 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32281 max_load_reduction_pct=0.493767 moved_to_replicas=96380 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=45 tokens_per_ep_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32351 max_load_reduction_pct=0.494308 moved_to_replicas=64491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=41 tokens_per_ep_rank=[32562, 33096, 32682, 32633, 33096, 32245, 32734, 33096] avg=32768.0000 max=33096 max_rank=1 min=32245 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32281 max_load_reduction_pct=0.493767 moved_to_replicas=96380 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=45 tokens_per_ep_rank=[32115, 64846, 32421, 236, 65447, 32556, 33486, 1037] avg=32768.0000 max=65447 max_rank=4 min=236 min_rank=3 imbalance_ratio=1.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=44 tokens_per_ep_rank=[39576, 89874, 159, 64868, 391, 66597, 544, 135] avg=32768.0000 max=89874 max_rank=1 min=135 min_rank=7 imbalance_ratio=2.742737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=46 tokens_per_ep_rank=[527, 33686, 64915, 31810, 32867, 36903, 154, 61282] avg=32768.0000 max=64915 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=44 tokens_per_ep_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] avg=32768.0000 max=33096 max_rank=0 min=31228 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56778 max_load_reduction_pct=0.631751 moved_to_replicas=128531 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=45 tokens_per_ep_rank=[32115, 64846, 32421, 236, 65447, 32556, 33486, 1037] avg=32768.0000 max=65447 max_rank=4 min=236 min_rank=3 imbalance_ratio=1.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=45 tokens_per_ep_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32351 max_load_reduction_pct=0.494308 moved_to_replicas=64491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=44 tokens_per_ep_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] avg=32768.0000 max=33096 max_rank=0 min=31228 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56778 max_load_reduction_pct=0.631751 moved_to_replicas=128531 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=46 tokens_per_ep_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] avg=32768.0000 max=33096 max_rank=1 min=31973 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31819 max_load_reduction_pct=0.490164 moved_to_replicas=64402 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=42 tokens_per_ep_rank=[65391, 30929, 377, 129834, 259, 272, 2670, 32412] avg=32768.0000 max=129834 max_rank=3 min=259 min_rank=4 imbalance_ratio=3.962219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=46 tokens_per_ep_rank=[527, 33686, 64915, 31810, 32867, 36903, 154, 61282] avg=32768.0000 max=64915 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=42 tokens_per_ep_rank=[65391, 30929, 377, 129834, 259, 272, 2670, 32412] avg=32768.0000 max=129834 max_rank=3 min=259 min_rank=4 imbalance_ratio=3.962219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=45 tokens_per_ep_rank=[32115, 64846, 32421, 236, 65447, 32556, 33486, 1037] avg=32768.0000 max=65447 max_rank=4 min=236 min_rank=3 imbalance_ratio=1.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=45 tokens_per_ep_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32351 max_load_reduction_pct=0.494308 moved_to_replicas=64491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] avg=32768.0000 max=33096 max_rank=0 min=32084 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96738 max_load_reduction_pct=0.745090 moved_to_replicas=129033 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=46 tokens_per_ep_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] avg=32768.0000 max=33096 max_rank=1 min=31973 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31819 max_load_reduction_pct=0.490164 moved_to_replicas=64402 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=42 tokens_per_ep_rank=[33096, 32798, 32084, 33096, 32797, 32765, 33096, 32412] avg=32768.0000 max=33096 max_rank=0 min=32084 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96738 max_load_reduction_pct=0.745090 moved_to_replicas=129033 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=46 tokens_per_ep_rank=[527, 33686, 64915, 31810, 32867, 36903, 154, 61282] avg=32768.0000 max=64915 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=45 tokens_per_ep_rank=[32115, 64846, 32421, 236, 65447, 32556, 33486, 1037] avg=32768.0000 max=65447 max_rank=4 min=236 min_rank=3 imbalance_ratio=1.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=47 tokens_per_ep_rank=[300, 33010, 275, 33060, 32684, 65460, 62410, 34945] avg=32768.0000 max=65460 max_rank=5 min=275 min_rank=2 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=45 tokens_per_ep_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32351 max_load_reduction_pct=0.494308 moved_to_replicas=64491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=46 tokens_per_ep_rank=[527, 33686, 64915, 31810, 32867, 36903, 154, 61282] avg=32768.0000 max=64915 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=46 tokens_per_ep_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] avg=32768.0000 max=33096 max_rank=1 min=31973 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31819 max_load_reduction_pct=0.490164 moved_to_replicas=64402 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=45 tokens_per_ep_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32351 max_load_reduction_pct=0.494308 moved_to_replicas=64491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=47 tokens_per_ep_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=63527 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 29973, 260, 38922, 32582, 63014, 31407] avg=32768.0000 max=63014 max_rank=6 min=260 min_rank=3 imbalance_ratio=1.923035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=47 tokens_per_ep_rank=[300, 33010, 275, 33060, 32684, 65460, 62410, 34945] avg=32768.0000 max=65460 max_rank=5 min=275 min_rank=2 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 29973, 260, 38922, 32582, 63014, 31407] avg=32768.0000 max=63014 max_rank=6 min=260 min_rank=3 imbalance_ratio=1.923035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=46 tokens_per_ep_rank=[527, 33686, 64915, 31810, 32867, 36903, 154, 61282] avg=32768.0000 max=64915 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=46 tokens_per_ep_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] avg=32768.0000 max=33096 max_rank=1 min=31973 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31819 max_load_reduction_pct=0.490164 moved_to_replicas=64402 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=47 tokens_per_ep_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=63527 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] avg=32768.0000 max=33096 max_rank=2 min=31407 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29918 max_load_reduction_pct=0.474783 moved_to_replicas=35744 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=43 tokens_per_ep_rank=[33025, 32961, 33096, 32881, 33096, 32582, 33096, 31407] avg=32768.0000 max=33096 max_rank=2 min=31407 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29918 max_load_reduction_pct=0.474783 moved_to_replicas=35744 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=47 tokens_per_ep_rank=[300, 33010, 275, 33060, 32684, 65460, 62410, 34945] avg=32768.0000 max=65460 max_rank=5 min=275 min_rank=2 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=46 tokens_per_ep_rank=[527, 33686, 64915, 31810, 32867, 36903, 154, 61282] avg=32768.0000 max=64915 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=48 tokens_per_ep_rank=[2565, 32715, 30433, 269, 97651, 513, 64752, 33246] avg=32768.0000 max=97651 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=46 tokens_per_ep_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] avg=32768.0000 max=33096 max_rank=1 min=31973 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31819 max_load_reduction_pct=0.490164 moved_to_replicas=64402 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=47 tokens_per_ep_rank=[300, 33010, 275, 33060, 32684, 65460, 62410, 34945] avg=32768.0000 max=65460 max_rank=5 min=275 min_rank=2 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=47 tokens_per_ep_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=63527 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=46 tokens_per_ep_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] avg=32768.0000 max=33096 max_rank=1 min=31973 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31819 max_load_reduction_pct=0.490164 moved_to_replicas=64402 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=48 tokens_per_ep_rank=[2565, 32715, 30433, 269, 97651, 513, 64752, 33246] avg=32768.0000 max=97651 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64555 max_load_reduction_pct=0.661079 moved_to_replicas=96361 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=44 tokens_per_ep_rank=[39576, 89874, 159, 64868, 391, 66597, 544, 135] avg=32768.0000 max=89874 max_rank=1 min=135 min_rank=7 imbalance_ratio=2.742737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=44 tokens_per_ep_rank=[39576, 89874, 159, 64868, 391, 66597, 544, 135] avg=32768.0000 max=89874 max_rank=1 min=135 min_rank=7 imbalance_ratio=2.742737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=47 tokens_per_ep_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=63527 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=47 tokens_per_ep_rank=[300, 33010, 275, 33060, 32684, 65460, 62410, 34945] avg=32768.0000 max=65460 max_rank=5 min=275 min_rank=2 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64555 max_load_reduction_pct=0.661079 moved_to_replicas=96361 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=44 tokens_per_ep_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] avg=32768.0000 max=33096 max_rank=0 min=31228 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56778 max_load_reduction_pct=0.631751 moved_to_replicas=128531 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=44 tokens_per_ep_rank=[33096, 33094, 31228, 33096, 32849, 33096, 33096, 32589] avg=32768.0000 max=33096 max_rank=0 min=31228 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56778 max_load_reduction_pct=0.631751 moved_to_replicas=128531 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=48 tokens_per_ep_rank=[2565, 32715, 30433, 269, 97651, 513, 64752, 33246] avg=32768.0000 max=97651 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=47 tokens_per_ep_rank=[300, 33010, 275, 33060, 32684, 65460, 62410, 34945] avg=32768.0000 max=65460 max_rank=5 min=275 min_rank=2 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=47 tokens_per_ep_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=63527 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=48 tokens_per_ep_rank=[2565, 32715, 30433, 269, 97651, 513, 64752, 33246] avg=32768.0000 max=97651 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64555 max_load_reduction_pct=0.661079 moved_to_replicas=96361 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=47 tokens_per_ep_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=63527 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=45 tokens_per_ep_rank=[32115, 64846, 32421, 236, 65447, 32556, 33486, 1037] avg=32768.0000 max=65447 max_rank=4 min=236 min_rank=3 imbalance_ratio=1.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=45 tokens_per_ep_rank=[32115, 64846, 32421, 236, 65447, 32556, 33486, 1037] avg=32768.0000 max=65447 max_rank=4 min=236 min_rank=3 imbalance_ratio=1.997284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64555 max_load_reduction_pct=0.661079 moved_to_replicas=96361 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=48 tokens_per_ep_rank=[2565, 32715, 30433, 269, 97651, 513, 64752, 33246] avg=32768.0000 max=97651 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=45 tokens_per_ep_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32351 max_load_reduction_pct=0.494308 moved_to_replicas=64491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=45 tokens_per_ep_rank=[32505, 33096, 32421, 32587, 33096, 32556, 33096, 32787] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32351 max_load_reduction_pct=0.494308 moved_to_replicas=64491 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=48 tokens_per_ep_rank=[2565, 32715, 30433, 269, 97651, 513, 64752, 33246] avg=32768.0000 max=97651 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64555 max_load_reduction_pct=0.661079 moved_to_replicas=96361 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64555 max_load_reduction_pct=0.661079 moved_to_replicas=96361 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=46 tokens_per_ep_rank=[527, 33686, 64915, 31810, 32867, 36903, 154, 61282] avg=32768.0000 max=64915 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=46 tokens_per_ep_rank=[527, 33686, 64915, 31810, 32867, 36903, 154, 61282] avg=32768.0000 max=64915 max_rank=2 min=154 min_rank=6 imbalance_ratio=1.981049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=46 tokens_per_ep_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] avg=32768.0000 max=33096 max_rank=1 min=31973 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31819 max_load_reduction_pct=0.490164 moved_to_replicas=64402 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=46 tokens_per_ep_rank=[32522, 33096, 33096, 32400, 32867, 33096, 31973, 33094] avg=32768.0000 max=33096 max_rank=1 min=31973 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31819 max_load_reduction_pct=0.490164 moved_to_replicas=64402 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=47 tokens_per_ep_rank=[300, 33010, 275, 33060, 32684, 65460, 62410, 34945] avg=32768.0000 max=65460 max_rank=5 min=275 min_rank=2 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=47 tokens_per_ep_rank=[300, 33010, 275, 33060, 32684, 65460, 62410, 34945] avg=32768.0000 max=65460 max_rank=5 min=275 min_rank=2 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=47 tokens_per_ep_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=63527 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=47 tokens_per_ep_rank=[31463, 33010, 32639, 33060, 32684, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=63527 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=48 tokens_per_ep_rank=[2565, 32715, 30433, 269, 97651, 513, 64752, 33246] avg=32768.0000 max=97651 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=48 tokens_per_ep_rank=[2565, 32715, 30433, 269, 97651, 513, 64752, 33246] avg=32768.0000 max=97651 max_rank=4 min=269 min_rank=3 imbalance_ratio=2.980072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64555 max_load_reduction_pct=0.661079 moved_to_replicas=96361 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=48 tokens_per_ep_rank=[33096, 32715, 31708, 32816, 33096, 32521, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64555 max_load_reduction_pct=0.661079 moved_to_replicas=96361 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=48 total_expected_recv=1569438 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=370304 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=48 total_expected_recv=1571456 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=367744 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=48 total_expected_recv=1574910 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365440 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=48 total_expected_recv=1573604 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368512 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=48 total_expected_recv=1572386 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=369280 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=48 total_expected_recv=1574275 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364672 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=48 total_expected_recv=1574639 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365440 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=48 total_expected_recv=1572204 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365056 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:47:01] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 714.7 | throughput per GPU (TFLOP/s/GPU): 124.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604127E+00 | loss scale: 1.0 | grad norm: 8.704 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=30912 expected_overflow=0 exact_recv_per_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=30912 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32233 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32233 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=31335 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32076 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=31335 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31994 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32076 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=30471 expected_overflow=0 exact_recv_per_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=30471 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=31492 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31994 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=31492 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34688 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31575 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=31575 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32073 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32073 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=31943 expected_overflow=0 exact_recv_per_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=31943 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32357 expected_overflow=0 exact_recv_per_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32357 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=31960 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32215 expected_overflow=0 exact_recv_per_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=31960 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32215 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32293 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32293 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=31850 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=31850 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32559 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=31770 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32941 expected_overflow=0 exact_recv_per_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=31778 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=31770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32941 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=31778 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=31730 expected_overflow=0 exact_recv_per_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=31730 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=31730 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=31730 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33054 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33054 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=31839 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=31839 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=31796 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=31796 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32330 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32455 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32330 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32277 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32277 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33059 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33059 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32260 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32223 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32260 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32223 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32327 expected_overflow=0 exact_recv_per_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32327 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32106 expected_overflow=0 exact_recv_per_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32106 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32382 expected_overflow=0 exact_recv_per_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32382 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32209 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32209 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32403 expected_overflow=0 exact_recv_per_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32403 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32321 expected_overflow=0 exact_recv_per_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32321 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=40960 expected_recv=31716 expected_overflow=0 exact_recv_per_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32356 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=40960 expected_recv=31709 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=40960 expected_recv=31709 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=40960 expected_recv=31716 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32356 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32411 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32383 expected_overflow=0 exact_recv_per_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32413 expected_overflow=0 exact_recv_per_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32411 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32383 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32413 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32400 expected_overflow=0 exact_recv_per_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32400 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32400 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32353 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32353 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32400 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32403 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32403 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32488 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32488 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32084 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32348 expected_overflow=0 exact_recv_per_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32087 expected_overflow=0 exact_recv_per_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32348 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32087 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32406 expected_overflow=0 exact_recv_per_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32499 expected_overflow=0 exact_recv_per_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32406 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32499 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32242 expected_overflow=0 exact_recv_per_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32394 expected_overflow=0 exact_recv_per_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33026 expected_overflow=0 exact_recv_per_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32242 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32394 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32405 expected_overflow=0 exact_recv_per_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32265 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32405 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32265 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33026 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32300 expected_overflow=0 exact_recv_per_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32300 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32307 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32310 expected_overflow=0 exact_recv_per_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32298 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32307 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32310 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32298 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32477 expected_overflow=0 exact_recv_per_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32477 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32304 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32304 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32479 expected_overflow=0 exact_recv_per_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32479 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32322 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32322 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32455 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32488 expected_overflow=0 exact_recv_per_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32488 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32332 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32332 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32296 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32296 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32326 expected_overflow=0 exact_recv_per_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32326 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32497 expected_overflow=0 exact_recv_per_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32293 expected_overflow=0 exact_recv_per_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32497 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32293 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32473 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32473 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32474 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32474 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32517 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32517 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32098 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32098 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32392 expected_overflow=0 exact_recv_per_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32397 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32395 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32397 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32395 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32392 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32102 expected_overflow=0 exact_recv_per_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32102 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32413 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32413 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32517 expected_overflow=0 exact_recv_per_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32517 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32490 expected_overflow=0 exact_recv_per_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32490 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32395 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32395 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32304 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32304 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32321 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32321 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32342 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32355 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32342 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32355 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32282 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32411 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32411 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32354 expected_overflow=0 exact_recv_per_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32354 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32300 expected_overflow=0 exact_recv_per_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32300 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32387 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32387 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32343 expected_overflow=0 exact_recv_per_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=36 calls=1 budget=40960 expected_recv=31850 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=36 calls=1 budget=40960 expected_recv=31850 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32362 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32343 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32317 expected_overflow=0 exact_recv_per_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32317 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=36 calls=1 budget=40960 expected_recv=31855 expected_overflow=0 exact_recv_per_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=36 calls=1 budget=40960 expected_recv=31855 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32362 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32279 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32363 expected_overflow=0 exact_recv_per_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32279 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32363 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32368 expected_overflow=0 exact_recv_per_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32368 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32412 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32459 expected_overflow=0 exact_recv_per_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32459 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32459 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32459 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32473 expected_overflow=0 exact_recv_per_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32473 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32412 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32461 expected_overflow=0 exact_recv_per_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32461 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32317 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32317 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32262 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32334 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32262 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32334 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32097 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32258 expected_overflow=0 exact_recv_per_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32097 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32258 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32390 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32936 expected_overflow=0 exact_recv_per_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32936 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32390 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32314 expected_overflow=0 exact_recv_per_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32314 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32338 expected_overflow=0 exact_recv_per_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32338 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32096 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32096 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32334 expected_overflow=0 exact_recv_per_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32334 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32424 expected_overflow=0 exact_recv_per_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32381 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32424 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32320 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32381 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32484 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32484 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32388 expected_overflow=0 exact_recv_per_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=45 calls=1 budget=40960 expected_recv=31453 expected_overflow=0 exact_recv_per_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32388 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=45 calls=1 budget=40960 expected_recv=31453 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=45 calls=1 budget=40960 expected_recv=31452 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=45 calls=1 budget=40960 expected_recv=31452 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32073 expected_overflow=0 exact_recv_per_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32073 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32381 expected_overflow=0 exact_recv_per_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32381 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[36766, 17085, 36362, 23530, 45465, 24726, 43657, 34553] avg=32768.0000 max=45465 max_rank=4 min=17085 min_rank=1 imbalance_ratio=1.387482 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32077 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30912 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=12369 max_load_reduction_pct=0.272055 moved_to_replicas=31323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[36766, 17085, 36362, 23530, 45465, 24726, 43657, 34553] avg=32768.0000 max=45465 max_rank=4 min=17085 min_rank=1 imbalance_ratio=1.387482 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32077 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[36766, 17085, 36362, 23530, 45465, 24726, 43657, 34553] avg=32768.0000 max=45465 max_rank=4 min=17085 min_rank=1 imbalance_ratio=1.387482 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[36766, 17085, 36362, 23530, 45465, 24726, 43657, 34553] avg=32768.0000 max=45465 max_rank=4 min=17085 min_rank=1 imbalance_ratio=1.387482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30912 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=12369 max_load_reduction_pct=0.272055 moved_to_replicas=31323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[19289, 38619, 47911, 811, 11795, 114295, 4623, 24801] avg=32768.0000 max=114295 max_rank=5 min=811 min_rank=3 imbalance_ratio=3.488007 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[36766, 17085, 36362, 23530, 45465, 24726, 43657, 34553] avg=32768.0000 max=45465 max_rank=4 min=17085 min_rank=1 imbalance_ratio=1.387482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30912 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=12369 max_load_reduction_pct=0.272055 moved_to_replicas=31323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30912 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=12369 max_load_reduction_pct=0.272055 moved_to_replicas=31323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] avg=32768.0000 max=33096 max_rank=0 min=31492 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=81199 max_load_reduction_pct=0.710434 moved_to_replicas=101537 active_clone_links=7 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[36637, 17082, 36250, 23357, 45658, 24795, 43772, 34593] avg=32768.0000 max=45658 max_rank=4 min=17082 min_rank=1 imbalance_ratio=1.393372 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33096, 31141, 33097, 33096, 33096, 32426, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31141 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12561 max_load_reduction_pct=0.275111 moved_to_replicas=31430 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[20352, 37864, 47697, 703, 11785, 114073, 4902, 24768] avg=32768.0000 max=114073 max_rank=5 min=703 min_rank=3 imbalance_ratio=3.481232 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31335, 33096, 33096, 33096, 32233] avg=32768.0000 max=33096 max_rank=0 min=31335 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=80977 max_load_reduction_pct=0.709870 moved_to_replicas=100346 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[9030, 7062, 36254, 52138, 99254, 10801, 1236, 46369] avg=32768.0000 max=99254 max_rank=4 min=1236 min_rank=6 imbalance_ratio=3.028992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[31994, 33094, 33096, 33096, 33096, 31575, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31575 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=66157 max_load_reduction_pct=0.666542 moved_to_replicas=101631 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[130, 13492, 32836, 105532, 32215, 33860, 32879, 11200] avg=32768.0000 max=105532 max_rank=3 min=130 min_rank=0 imbalance_ratio=3.220581 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32854, 32073, 32836, 33095, 32215, 33096, 32879, 33096] avg=32768.0000 max=33096 max_rank=5 min=32073 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72436 max_load_reduction_pct=0.686389 moved_to_replicas=73200 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[84383, 32759, 46764, 29433, 29, 57, 37394, 31325] avg=32768.0000 max=84383 max_rank=0 min=29 min_rank=4 imbalance_ratio=2.575165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33095, 33096, 32748, 32293, 33096, 31960] avg=32768.0000 max=33097 max_rank=0 min=31960 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51286 max_load_reduction_pct=0.607776 moved_to_replicas=69253 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[46, 65432, 30650, 32762, 32755, 73, 32787, 67639] avg=32768.0000 max=67639 max_rank=7 min=46 min_rank=0 imbalance_ratio=2.064178 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32774, 33096, 31778, 32762, 32755, 33096, 32787, 33096] avg=32768.0000 max=33096 max_rank=1 min=31778 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34543 max_load_reduction_pct=0.510696 moved_to_replicas=66879 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[41286, 32758, 32833, 32805, 4, 80728, 7536, 34194] avg=32768.0000 max=80728 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.463623 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[33097, 32758, 32833, 32805, 32729, 33096, 31730, 33096] avg=32768.0000 max=33097 max_rank=0 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=47631 max_load_reduction_pct=0.590018 moved_to_replicas=56920 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[66764, 32837, 31839, 32762, 14, 65144, 32729, 55] avg=32768.0000 max=66764 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.037476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31839, 32762, 32731, 33096, 32729, 33054] avg=32768.0000 max=33096 max_rank=0 min=31839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33668 max_load_reduction_pct=0.504284 moved_to_replicas=65716 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32799, 66, 65485, 32802, 32810, 22377, 32756, 43049] avg=32768.0000 max=65485 max_rank=2 min=66 min_rank=1 imbalance_ratio=1.998444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32799, 32455, 33096, 32802, 32810, 32330, 32756, 33096] avg=32768.0000 max=33096 max_rank=2 min=32330 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32389 max_load_reduction_pct=0.494602 moved_to_replicas=42342 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[23, 102, 15, 273, 130715, 33059, 97807, 150] avg=32768.0000 max=130715 max_rank=4 min=15 min_rank=2 imbalance_ratio=3.989105 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32744, 32277, 32738, 32260, 33096, 33059, 33096, 32874] avg=32768.0000 max=33096 max_rank=4 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=97619 max_load_reduction_pct=0.746808 moved_to_replicas=162330 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[64, 36743, 32727, 32738, 32781, 32712, 81239, 13140] avg=32768.0000 max=81239 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.479218 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32770, 33096, 32727, 32738, 32781, 32712, 33097, 32223] avg=32768.0000 max=33097 max_rank=6 min=32223 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=48142 max_load_reduction_pct=0.592597 moved_to_replicas=51790 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[83, 32781, 65539, 65461, 32321, 32755, 32705, 499] avg=32768.0000 max=65539 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32526, 32781, 33096, 33096, 32321, 32755, 32705, 32864] avg=32768.0000 max=33096 max_rank=2 min=32321 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=64808 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[9, 32846, 1008, 65481, 32800, 89096, 8136, 32768] avg=32768.0000 max=89096 max_rank=5 min=9 min_rank=0 imbalance_ratio=2.718994 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32846, 31709, 33096, 32800, 33097, 33096, 32768] avg=32768.0000 max=33097 max_rank=5 min=31709 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55999 max_load_reduction_pct=0.628524 moved_to_replicas=88385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32759, 98123, 65395, 32811, 42, 129, 32871, 14] avg=32768.0000 max=98123 max_rank=1 min=14 min_rank=7 imbalance_ratio=2.994476 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32759, 33096, 33096, 32811, 32353, 32428, 32871, 32730] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65027 max_load_reduction_pct=0.662709 moved_to_replicas=97326 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[100, 65490, 65371, 65488, 27, 32773, 32889, 6] avg=32768.0000 max=65490 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32375, 33096, 33096, 33096, 32419, 32773, 32889, 32400] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=97061 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[47729, 32758, 32787, 32403, 65715, 18037, 53, 32662] avg=32768.0000 max=65715 max_rank=4 min=53 min_rank=6 imbalance_ratio=2.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[33096, 32758, 32787, 32403, 33096, 32670, 32672, 32662] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32619 max_load_reduction_pct=0.496371 moved_to_replicas=47252 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[32814, 111, 32737, 32767, 65473, 539, 64915, 32788] avg=32768.0000 max=65473 max_rank=4 min=111 min_rank=1 imbalance_ratio=1.998077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[32814, 32488, 32737, 32767, 33096, 32358, 33096, 32788] avg=32768.0000 max=33096 max_rank=4 min=32358 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=64196 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[12, 46, 56, 32777, 65540, 66769, 32763, 64181] avg=32768.0000 max=66769 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.037628 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32084, 32500, 32777, 33096, 33096, 32763, 33096] avg=32768.0000 max=33096 max_rank=4 min=32084 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33673 max_load_reduction_pct=0.504321 moved_to_replicas=97202 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 42, 32808, 39, 65462, 32824, 65450] avg=32768.0000 max=65462 max_rank=5 min=39 min_rank=4 imbalance_ratio=1.997742 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32781, 32396, 32808, 32405, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32396 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64720 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[65152, 209, 32779, 166, 98, 32774, 98011, 32955] avg=32768.0000 max=98011 max_rank=6 min=98 min_rank=4 imbalance_ratio=2.991058 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32265, 32779, 32358, 32821, 32774, 33096, 32955] avg=32768.0000 max=33096 max_rank=0 min=32265 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64915 max_load_reduction_pct=0.662324 moved_to_replicas=96971 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[32805, 33, 32773, 81444, 75, 49824, 32438, 32752] avg=32768.0000 max=81444 max_rank=3 min=33 min_rank=1 imbalance_ratio=2.485474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32756, 32773, 33097, 32428, 33095, 32438, 32752] avg=32768.0000 max=33097 max_rank=3 min=32428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=48347 max_load_reduction_pct=0.593623 moved_to_replicas=65076 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[34216, 64124, 32622, 156, 87, 32814, 98034, 91] avg=32768.0000 max=98034 max_rank=6 min=87 min_rank=4 imbalance_ratio=2.991760 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32304, 32809, 32814, 33096, 32307] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64938 max_load_reduction_pct=0.662403 moved_to_replicas=97086 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[32741, 73, 32768, 98224, 32734, 331, 33, 65240] avg=32768.0000 max=98224 max_rank=3 min=33 min_rank=6 imbalance_ratio=2.997559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[32741, 32478, 32768, 33096, 32734, 32475, 32756, 33096] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[110, 32724, 65567, 26, 32715, 32721, 98209, 72] avg=32768.0000 max=98209 max_rank=6 min=26 min_rank=3 imbalance_ratio=2.997101 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[32581, 32724, 33096, 32733, 32715, 32721, 33096, 32478] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65113 max_load_reduction_pct=0.663004 moved_to_replicas=97584 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=25 tokens_per_ep_rank=[98, 72, 32751, 98177, 26693, 38886, 32691, 32776] avg=32768.0000 max=98177 max_rank=3 min=72 min_rank=1 imbalance_ratio=2.996124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=25 tokens_per_ep_rank=[32456, 32795, 32751, 33096, 32485, 33094, 32691, 32776] avg=32768.0000 max=33096 max_rank=3 min=32456 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70871 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=26 tokens_per_ep_rank=[32788, 31, 33689, 45, 64754, 32698, 65397, 32742] avg=32768.0000 max=65397 max_rank=6 min=31 min_rank=1 imbalance_ratio=1.995758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=26 tokens_per_ep_rank=[32788, 32332, 33096, 32296, 33096, 32698, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32296 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32301 max_load_reduction_pct=0.493922 moved_to_replicas=64552 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 78, 32857, 32799, 65493, 32740] avg=32768.0000 max=65493 max_rank=6 min=78 min_rank=3 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=27 tokens_per_ep_rank=[32806, 32605, 32766, 32475, 32857, 32799, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=32397 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=28 tokens_per_ep_rank=[32702, 65449, 58, 32761, 65477, 85, 89, 65523] avg=32768.0000 max=65523 max_rank=7 min=58 min_rank=2 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=28 tokens_per_ep_rank=[32702, 33096, 32485, 32761, 33096, 32466, 32442, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=97161 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=29 tokens_per_ep_rank=[115, 65516, 65503, 32723, 65477, 15, 32781, 14] avg=32768.0000 max=65516 max_rank=1 min=14 min_rank=7 imbalance_ratio=1.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=29 tokens_per_ep_rank=[32496, 33096, 33096, 32723, 33096, 32422, 32781, 32434] avg=32768.0000 max=33096 max_rank=1 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97208 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=30 tokens_per_ep_rank=[47, 65443, 32723, 32801, 32797, 65479, 32728, 126] avg=32768.0000 max=65479 max_rank=5 min=47 min_rank=0 imbalance_ratio=1.998260 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32723, 32801, 32797, 33096, 32728, 32473] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32383 max_load_reduction_pct=0.494556 moved_to_replicas=64730 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=31 tokens_per_ep_rank=[32806, 65486, 32517, 65490, 1, 32793, 32471, 580] avg=32768.0000 max=65490 max_rank=3 min=1 min_rank=4 imbalance_ratio=1.998596 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=31 tokens_per_ep_rank=[32806, 33096, 32517, 33096, 32395, 32793, 32471, 32970] avg=32768.0000 max=33096 max_rank=1 min=32395 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=64784 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=32 tokens_per_ep_rank=[12920, 512, 32757, 98225, 32787, 78, 6, 84859] avg=32768.0000 max=98225 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.997589 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=32 tokens_per_ep_rank=[32098, 33096, 32757, 33096, 32787, 32492, 32721, 33097] avg=32768.0000 max=33097 max_rank=7 min=32098 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65128 max_load_reduction_pct=0.663049 moved_to_replicas=116892 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=33 tokens_per_ep_rank=[65466, 65444, 43, 32787, 32732, 32802, 32821, 49] avg=32768.0000 max=65466 max_rank=0 min=43 min_rank=2 imbalance_ratio=1.997864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32413, 32787, 32732, 32802, 32821, 32397] avg=32768.0000 max=33096 max_rank=0 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32370 max_load_reduction_pct=0.494455 moved_to_replicas=64718 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=34 tokens_per_ep_rank=[98018, 89, 103, 56, 32823, 7, 32970, 98078] avg=32768.0000 max=98078 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.993103 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32810, 32304, 32321, 32823, 32724, 32970, 33096] avg=32768.0000 max=33096 max_rank=0 min=32304 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64982 max_load_reduction_pct=0.662554 moved_to_replicas=129904 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=35 tokens_per_ep_rank=[32869, 59, 32807, 32798, 65395, 56, 32781, 65379] avg=32768.0000 max=65395 max_rank=4 min=56 min_rank=5 imbalance_ratio=1.995697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=35 tokens_per_ep_rank=[32869, 32342, 32807, 32798, 33096, 32355, 32781, 33096] avg=32768.0000 max=33096 max_rank=4 min=32342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32299 max_load_reduction_pct=0.493906 moved_to_replicas=64582 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=36 tokens_per_ep_rank=[65359, 65749, 26, 12, 99, 87, 98073, 32739] avg=32768.0000 max=98073 max_rank=6 min=12 min_rank=3 imbalance_ratio=2.992950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32282, 32733, 32362, 32740, 33096, 32739] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64977 max_load_reduction_pct=0.662537 moved_to_replicas=129893 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=37 tokens_per_ep_rank=[65418, 101, 126, 65436, 1063, 2, 9, 129989] avg=32768.0000 max=129989 max_rank=7 min=2 min_rank=5 imbalance_ratio=3.966949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31850, 32466, 33096, 33096, 32719, 32725, 33096] avg=32768.0000 max=33096 max_rank=0 min=31850 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96893 max_load_reduction_pct=0.745394 moved_to_replicas=161555 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=38 tokens_per_ep_rank=[32830, 65442, 136, 65420, 125, 0, 39, 98152] avg=32768.0000 max=98152 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.995361 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=38 tokens_per_ep_rank=[32830, 33096, 32460, 33096, 32471, 32720, 32375, 33096] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65056 max_load_reduction_pct=0.662809 moved_to_replicas=129726 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 42, 453, 32765, 32860, 98150, 32740] avg=32768.0000 max=98150 max_rank=6 min=42 min_rank=2 imbalance_ratio=2.995300 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=39 tokens_per_ep_rank=[32412, 32722, 32760, 32789, 32765, 32860, 33096, 32740] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 30697, 32731, 67572, 30] avg=32768.0000 max=67572 max_rank=6 min=30 min_rank=7 imbalance_ratio=2.062134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=40 tokens_per_ep_rank=[32796, 32738, 32768, 32812, 32453, 32731, 33096, 32750] avg=32768.0000 max=33096 max_rank=6 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34476 max_load_reduction_pct=0.510211 moved_to_replicas=34476 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=41 tokens_per_ep_rank=[67, 65488, 32812, 65222, 65491, 1, 136, 32927] avg=32768.0000 max=65491 max_rank=4 min=1 min_rank=5 imbalance_ratio=1.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=41 tokens_per_ep_rank=[32459, 33096, 32812, 33096, 33096, 32396, 32262, 32927] avg=32768.0000 max=33096 max_rank=1 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32395 max_load_reduction_pct=0.494648 moved_to_replicas=96913 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=42 tokens_per_ep_rank=[65692, 0, 93, 32783, 65355, 65337, 32826, 58] avg=32768.0000 max=65692 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32596, 32334, 32783, 33096, 33096, 32826, 32317] avg=32768.0000 max=33096 max_rank=0 min=32317 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32596 max_load_reduction_pct=0.496194 moved_to_replicas=97096 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=43 tokens_per_ep_rank=[53, 129916, 69, 105, 32806, 29, 32725, 66441] avg=32768.0000 max=129916 max_rank=1 min=29 min_rank=5 imbalance_ratio=3.964722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32097, 32816, 32806, 32746, 32725, 33096] avg=32768.0000 max=33096 max_rank=1 min=32097 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96820 max_load_reduction_pct=0.745251 moved_to_replicas=130165 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 12, 32742, 46, 65474, 32866, 65474] avg=32768.0000 max=65474 max_rank=5 min=12 min_rank=2 imbalance_ratio=1.998108 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=44 tokens_per_ep_rank=[32746, 32784, 32390, 32742, 32424, 33096, 32866, 33096] avg=32768.0000 max=33096 max_rank=5 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=45 tokens_per_ep_rank=[32801, 31504, 105, 4, 99720, 32877, 32320, 32813] avg=32768.0000 max=99720 max_rank=4 min=4 min_rank=3 imbalance_ratio=3.043213 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32699, 32816, 32722, 33096, 32877, 32320, 32813] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66624 max_load_reduction_pct=0.668111 moved_to_replicas=66624 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=46 tokens_per_ep_rank=[10715, 92, 65495, 3, 23137, 97180, 51, 65471] avg=32768.0000 max=97180 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.965698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=46 tokens_per_ep_rank=[33096, 32491, 33096, 32721, 33096, 33096, 31452, 33096] avg=32768.0000 max=33096 max_rank=0 min=31452 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64084 max_load_reduction_pct=0.659436 moved_to_replicas=128858 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=47 tokens_per_ep_rank=[87, 32852, 6, 32720, 32792, 65493, 98121, 73] avg=32768.0000 max=98121 max_rank=6 min=6 min_rank=2 imbalance_ratio=2.994415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=47 tokens_per_ep_rank=[32484, 32852, 32723, 32720, 32792, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65025 max_load_reduction_pct=0.662702 moved_to_replicas=97422 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=48 tokens_per_ep_rank=[61, 32730, 32751, 32719, 61249, 32798, 36958, 32878] avg=32768.0000 max=61249 max_rank=4 min=61 min_rank=0 imbalance_ratio=1.869171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=48 tokens_per_ep_rank=[32077, 32730, 32751, 32719, 33095, 32798, 33096, 32878] avg=32768.0000 max=33096 max_rank=6 min=32077 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28153 max_load_reduction_pct=0.459648 moved_to_replicas=32015 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=48 total_expected_recv=1574855 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=361856 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=48 total_expected_recv=1572739 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364416 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=48 total_expected_recv=1572902 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365696 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=48 total_expected_recv=1573384 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366848 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=48 total_expected_recv=1569156 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=370688 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=48 total_expected_recv=1571237 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368640 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=48 total_expected_recv=1574485 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364928 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=48 total_expected_recv=1574154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364672 peak_demand=33096 required_capacity=1.010010 - [2026-08-26 15:01:22] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 715.5 | throughput per GPU (TFLOP/s/GPU): 124.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307577E+00 | loss scale: 1.0 | grad norm: 9.631 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32348 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31220 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32348 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[19289, 38619, 47911, 811, 11795, 114295, 4623, 24801] avg=32768.0000 max=114295 max_rank=5 min=811 min_rank=3 imbalance_ratio=3.488007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[36766, 17085, 36362, 23530, 45465, 24726, 43657, 34553] avg=32768.0000 max=45465 max_rank=4 min=17085 min_rank=1 imbalance_ratio=1.387482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30912 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=12369 max_load_reduction_pct=0.272055 moved_to_replicas=31323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[19289, 38619, 47911, 811, 11795, 114295, 4623, 24801] avg=32768.0000 max=114295 max_rank=5 min=811 min_rank=3 imbalance_ratio=3.488007 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] avg=32768.0000 max=33096 max_rank=0 min=31492 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=81199 max_load_reduction_pct=0.710434 moved_to_replicas=101537 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[19289, 38619, 47911, 811, 11795, 114295, 4623, 24801] avg=32768.0000 max=114295 max_rank=5 min=811 min_rank=3 imbalance_ratio=3.488007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[9906, 7887, 36183, 49825, 99262, 12047, 906, 46128] avg=32768.0000 max=99262 max_rank=4 min=906 min_rank=6 imbalance_ratio=3.029236 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30912 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=12369 max_load_reduction_pct=0.272055 moved_to_replicas=31323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[36766, 17085, 36362, 23530, 45465, 24726, 43657, 34553] avg=32768.0000 max=45465 max_rank=4 min=17085 min_rank=1 imbalance_ratio=1.387482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] avg=32768.0000 max=33096 max_rank=0 min=31492 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=81199 max_load_reduction_pct=0.710434 moved_to_replicas=101537 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=30471 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=66165 max_load_reduction_pct=0.666569 moved_to_replicas=99014 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[19289, 38619, 47911, 811, 11795, 114295, 4623, 24801] avg=32768.0000 max=114295 max_rank=5 min=811 min_rank=3 imbalance_ratio=3.488007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] avg=32768.0000 max=33096 max_rank=0 min=31492 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=81199 max_load_reduction_pct=0.710434 moved_to_replicas=101537 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[9906, 7887, 36183, 49825, 99262, 12047, 906, 46128] avg=32768.0000 max=99262 max_rank=4 min=906 min_rank=6 imbalance_ratio=3.029236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30912 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=12369 max_load_reduction_pct=0.272055 moved_to_replicas=31323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[19289, 38619, 47911, 811, 11795, 114295, 4623, 24801] avg=32768.0000 max=114295 max_rank=5 min=811 min_rank=3 imbalance_ratio=3.488007 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] avg=32768.0000 max=33096 max_rank=0 min=31492 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=81199 max_load_reduction_pct=0.710434 moved_to_replicas=101537 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[9906, 7887, 36183, 49825, 99262, 12047, 906, 46128] avg=32768.0000 max=99262 max_rank=4 min=906 min_rank=6 imbalance_ratio=3.029236 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=30471 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=66165 max_load_reduction_pct=0.666569 moved_to_replicas=99014 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[113, 14616, 32842, 103637, 32357, 33680, 32877, 12022] avg=32768.0000 max=103637 max_rank=3 min=113 min_rank=0 imbalance_ratio=3.162750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] avg=32768.0000 max=33096 max_rank=0 min=31492 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=81199 max_load_reduction_pct=0.710434 moved_to_replicas=101537 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[9906, 7887, 36183, 49825, 99262, 12047, 906, 46128] avg=32768.0000 max=99262 max_rank=4 min=906 min_rank=6 imbalance_ratio=3.029236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[19289, 38619, 47911, 811, 11795, 114295, 4623, 24801] avg=32768.0000 max=114295 max_rank=5 min=811 min_rank=3 imbalance_ratio=3.488007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=30471 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=66165 max_load_reduction_pct=0.666569 moved_to_replicas=99014 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=3 min=31943 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=70541 max_load_reduction_pct=0.680655 moved_to_replicas=71125 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[9906, 7887, 36183, 49825, 99262, 12047, 906, 46128] avg=32768.0000 max=99262 max_rank=4 min=906 min_rank=6 imbalance_ratio=3.029236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=30471 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=66165 max_load_reduction_pct=0.666569 moved_to_replicas=99014 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[113, 14616, 32842, 103637, 32357, 33680, 32877, 12022] avg=32768.0000 max=103637 max_rank=3 min=113 min_rank=0 imbalance_ratio=3.162750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[36766, 17085, 36362, 23530, 45465, 24726, 43657, 34553] avg=32768.0000 max=45465 max_rank=4 min=17085 min_rank=1 imbalance_ratio=1.387482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] avg=32768.0000 max=33096 max_rank=0 min=31492 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=81199 max_load_reduction_pct=0.710434 moved_to_replicas=101537 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[9906, 7887, 36183, 49825, 99262, 12047, 906, 46128] avg=32768.0000 max=99262 max_rank=4 min=906 min_rank=6 imbalance_ratio=3.029236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=30471 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=66165 max_load_reduction_pct=0.666569 moved_to_replicas=99014 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[113, 14616, 32842, 103637, 32357, 33680, 32877, 12022] avg=32768.0000 max=103637 max_rank=3 min=113 min_rank=0 imbalance_ratio=3.162750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=3 min=31943 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=70541 max_load_reduction_pct=0.680655 moved_to_replicas=71125 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[84335, 32759, 47077, 29001, 28, 58, 37036, 31850] avg=32768.0000 max=84335 max_rank=0 min=28 min_rank=4 imbalance_ratio=2.573700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30912, 33096, 33096, 33095, 32657, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30912 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=12369 max_load_reduction_pct=0.272055 moved_to_replicas=31323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=30471 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=66165 max_load_reduction_pct=0.666569 moved_to_replicas=99014 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[113, 14616, 32842, 103637, 32357, 33680, 32877, 12022] avg=32768.0000 max=103637 max_rank=3 min=113 min_rank=0 imbalance_ratio=3.162750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[9906, 7887, 36183, 49825, 99262, 12047, 906, 46128] avg=32768.0000 max=99262 max_rank=4 min=906 min_rank=6 imbalance_ratio=3.029236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=3 min=31943 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=70541 max_load_reduction_pct=0.680655 moved_to_replicas=71125 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] avg=32768.0000 max=33097 max_rank=0 min=31850 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51238 max_load_reduction_pct=0.607553 moved_to_replicas=69160 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[113, 14616, 32842, 103637, 32357, 33680, 32877, 12022] avg=32768.0000 max=103637 max_rank=3 min=113 min_rank=0 imbalance_ratio=3.162750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[84335, 32759, 47077, 29001, 28, 58, 37036, 31850] avg=32768.0000 max=84335 max_rank=0 min=28 min_rank=4 imbalance_ratio=2.573700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=3 min=31943 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=70541 max_load_reduction_pct=0.680655 moved_to_replicas=71125 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=30471 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=66165 max_load_reduction_pct=0.666569 moved_to_replicas=99014 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[113, 14616, 32842, 103637, 32357, 33680, 32877, 12022] avg=32768.0000 max=103637 max_rank=3 min=113 min_rank=0 imbalance_ratio=3.162750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[19289, 38619, 47911, 811, 11795, 114295, 4623, 24801] avg=32768.0000 max=114295 max_rank=5 min=811 min_rank=3 imbalance_ratio=3.488007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=3 min=31943 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=70541 max_load_reduction_pct=0.680655 moved_to_replicas=71125 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[84335, 32759, 47077, 29001, 28, 58, 37036, 31850] avg=32768.0000 max=84335 max_rank=0 min=28 min_rank=4 imbalance_ratio=2.573700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[50, 65436, 30819, 32761, 32762, 73, 32785, 67458] avg=32768.0000 max=67458 max_rank=7 min=50 min_rank=0 imbalance_ratio=2.058655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] avg=32768.0000 max=33097 max_rank=0 min=31850 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51238 max_load_reduction_pct=0.607553 moved_to_replicas=69160 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=3 min=31943 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=70541 max_load_reduction_pct=0.680655 moved_to_replicas=71125 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[84335, 32759, 47077, 29001, 28, 58, 37036, 31850] avg=32768.0000 max=84335 max_rank=0 min=28 min_rank=4 imbalance_ratio=2.573700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31492, 33096, 33096, 33096, 32076] avg=32768.0000 max=33096 max_rank=0 min=31492 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=81199 max_load_reduction_pct=0.710434 moved_to_replicas=101537 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[113, 14616, 32842, 103637, 32357, 33680, 32877, 12022] avg=32768.0000 max=103637 max_rank=3 min=113 min_rank=0 imbalance_ratio=3.162750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] avg=32768.0000 max=33097 max_rank=0 min=31850 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51238 max_load_reduction_pct=0.607553 moved_to_replicas=69160 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] avg=32768.0000 max=33096 max_rank=1 min=31770 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34362 max_load_reduction_pct=0.509384 moved_to_replicas=66702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[84335, 32759, 47077, 29001, 28, 58, 37036, 31850] avg=32768.0000 max=84335 max_rank=0 min=28 min_rank=4 imbalance_ratio=2.573700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[50, 65436, 30819, 32761, 32762, 73, 32785, 67458] avg=32768.0000 max=67458 max_rank=7 min=50 min_rank=0 imbalance_ratio=2.058655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] avg=32768.0000 max=33097 max_rank=0 min=31850 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51238 max_load_reduction_pct=0.607553 moved_to_replicas=69160 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=3 min=31943 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=70541 max_load_reduction_pct=0.680655 moved_to_replicas=71125 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[84335, 32759, 47077, 29001, 28, 58, 37036, 31850] avg=32768.0000 max=84335 max_rank=0 min=28 min_rank=4 imbalance_ratio=2.573700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[9906, 7887, 36183, 49825, 99262, 12047, 906, 46128] avg=32768.0000 max=99262 max_rank=4 min=906 min_rank=6 imbalance_ratio=3.029236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[44370, 32761, 32829, 32808, 3, 76401, 9049, 33923] avg=32768.0000 max=76401 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.331573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] avg=32768.0000 max=33097 max_rank=0 min=31850 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51238 max_load_reduction_pct=0.607553 moved_to_replicas=69160 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[50, 65436, 30819, 32761, 32762, 73, 32785, 67458] avg=32768.0000 max=67458 max_rank=7 min=50 min_rank=0 imbalance_ratio=2.058655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] avg=32768.0000 max=33096 max_rank=1 min=31770 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34362 max_load_reduction_pct=0.509384 moved_to_replicas=66702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] avg=32768.0000 max=33097 max_rank=0 min=31850 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51238 max_load_reduction_pct=0.607553 moved_to_replicas=69160 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[50, 65436, 30819, 32761, 32762, 73, 32785, 67458] avg=32768.0000 max=67458 max_rank=7 min=50 min_rank=0 imbalance_ratio=2.058655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[84335, 32759, 47077, 29001, 28, 58, 37036, 31850] avg=32768.0000 max=84335 max_rank=0 min=28 min_rank=4 imbalance_ratio=2.573700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[30471, 33096, 33096, 33097, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=30471 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=66165 max_load_reduction_pct=0.666569 moved_to_replicas=99014 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43304 max_load_reduction_pct=0.566799 moved_to_replicas=55406 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] avg=32768.0000 max=33096 max_rank=1 min=31770 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34362 max_load_reduction_pct=0.509384 moved_to_replicas=66702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[50, 65436, 30819, 32761, 32762, 73, 32785, 67458] avg=32768.0000 max=67458 max_rank=7 min=50 min_rank=0 imbalance_ratio=2.058655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[44370, 32761, 32829, 32808, 3, 76401, 9049, 33923] avg=32768.0000 max=76401 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.331573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] avg=32768.0000 max=33096 max_rank=1 min=31770 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34362 max_load_reduction_pct=0.509384 moved_to_replicas=66702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] avg=32768.0000 max=33097 max_rank=0 min=31850 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51238 max_load_reduction_pct=0.607553 moved_to_replicas=69160 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[50, 65436, 30819, 32761, 32762, 73, 32785, 67458] avg=32768.0000 max=67458 max_rank=7 min=50 min_rank=0 imbalance_ratio=2.058655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[66863, 32837, 31759, 32762, 14, 65125, 32730, 54] avg=32768.0000 max=66863 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.040497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[113, 14616, 32842, 103637, 32357, 33680, 32877, 12022] avg=32768.0000 max=103637 max_rank=3 min=113 min_rank=0 imbalance_ratio=3.162750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] avg=32768.0000 max=33096 max_rank=1 min=31770 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34362 max_load_reduction_pct=0.509384 moved_to_replicas=66702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43304 max_load_reduction_pct=0.566799 moved_to_replicas=55406 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[44370, 32761, 32829, 32808, 3, 76401, 9049, 33923] avg=32768.0000 max=76401 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.331573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] avg=32768.0000 max=33096 max_rank=1 min=31770 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34362 max_load_reduction_pct=0.509384 moved_to_replicas=66702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[44370, 32761, 32829, 32808, 3, 76401, 9049, 33923] avg=32768.0000 max=76401 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.331573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] avg=32768.0000 max=33096 max_rank=0 min=31796 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33767 max_load_reduction_pct=0.505018 moved_to_replicas=65796 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[50, 65436, 30819, 32761, 32762, 73, 32785, 67458] avg=32768.0000 max=67458 max_rank=7 min=50 min_rank=0 imbalance_ratio=2.058655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32837, 31943, 32842, 33096, 32357, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=3 min=31943 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=70541 max_load_reduction_pct=0.680655 moved_to_replicas=71125 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43304 max_load_reduction_pct=0.566799 moved_to_replicas=55406 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[66863, 32837, 31759, 32762, 14, 65125, 32730, 54] avg=32768.0000 max=66863 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.040497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[44370, 32761, 32829, 32808, 3, 76401, 9049, 33923] avg=32768.0000 max=76401 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.331573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43304 max_load_reduction_pct=0.566799 moved_to_replicas=55406 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] avg=32768.0000 max=33096 max_rank=1 min=31770 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34362 max_load_reduction_pct=0.509384 moved_to_replicas=66702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[44370, 32761, 32829, 32808, 3, 76401, 9049, 33923] avg=32768.0000 max=76401 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.331573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32802, 78, 65482, 32795, 32809, 20007, 32755, 45416] avg=32768.0000 max=65482 max_rank=2 min=78 min_rank=1 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] avg=32768.0000 max=33096 max_rank=0 min=31796 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33767 max_load_reduction_pct=0.505018 moved_to_replicas=65796 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43304 max_load_reduction_pct=0.566799 moved_to_replicas=55406 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[66863, 32837, 31759, 32762, 14, 65125, 32730, 54] avg=32768.0000 max=66863 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.040497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[84335, 32759, 47077, 29001, 28, 58, 37036, 31850] avg=32768.0000 max=84335 max_rank=0 min=28 min_rank=4 imbalance_ratio=2.573700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43304 max_load_reduction_pct=0.566799 moved_to_replicas=55406 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[66863, 32837, 31759, 32762, 14, 65125, 32730, 54] avg=32768.0000 max=66863 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.040497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] avg=32768.0000 max=33096 max_rank=2 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=44706 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[44370, 32761, 32829, 32808, 3, 76401, 9049, 33923] avg=32768.0000 max=76401 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.331573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] avg=32768.0000 max=33096 max_rank=0 min=31796 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33767 max_load_reduction_pct=0.505018 moved_to_replicas=65796 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[33097, 32759, 33096, 32941, 32746, 32559, 33096, 31850] avg=32768.0000 max=33097 max_rank=0 min=31850 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=51238 max_load_reduction_pct=0.607553 moved_to_replicas=69160 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32802, 78, 65482, 32795, 32809, 20007, 32755, 45416] avg=32768.0000 max=65482 max_rank=2 min=78 min_rank=1 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[66863, 32837, 31759, 32762, 14, 65125, 32730, 54] avg=32768.0000 max=66863 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.040497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] avg=32768.0000 max=33096 max_rank=0 min=31796 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33767 max_load_reduction_pct=0.505018 moved_to_replicas=65796 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[66863, 32837, 31759, 32762, 14, 65125, 32730, 54] avg=32768.0000 max=66863 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.040497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43304 max_load_reduction_pct=0.566799 moved_to_replicas=55406 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[23, 90, 20, 478, 130556, 32881, 97433, 663] avg=32768.0000 max=130556 max_rank=4 min=20 min_rank=2 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] avg=32768.0000 max=33096 max_rank=2 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=44706 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] avg=32768.0000 max=33096 max_rank=0 min=31796 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33767 max_load_reduction_pct=0.505018 moved_to_replicas=65796 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32802, 78, 65482, 32795, 32809, 20007, 32755, 45416] avg=32768.0000 max=65482 max_rank=2 min=78 min_rank=1 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[50, 65436, 30819, 32761, 32762, 73, 32785, 67458] avg=32768.0000 max=67458 max_rank=7 min=50 min_rank=0 imbalance_ratio=2.058655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] avg=32768.0000 max=33096 max_rank=0 min=31796 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33767 max_load_reduction_pct=0.505018 moved_to_replicas=65796 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32802, 78, 65482, 32795, 32809, 20007, 32755, 45416] avg=32768.0000 max=65482 max_rank=2 min=78 min_rank=1 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] avg=32768.0000 max=33096 max_rank=3 min=32106 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97460 max_load_reduction_pct=0.746500 moved_to_replicas=161797 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[66863, 32837, 31759, 32762, 14, 65125, 32730, 54] avg=32768.0000 max=66863 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.040497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] avg=32768.0000 max=33096 max_rank=2 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=44706 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[23, 90, 20, 478, 130556, 32881, 97433, 663] avg=32768.0000 max=130556 max_rank=4 min=20 min_rank=2 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32778, 33096, 31770, 32761, 32762, 33096, 32785, 33096] avg=32768.0000 max=33096 max_rank=1 min=31770 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34362 max_load_reduction_pct=0.509384 moved_to_replicas=66702 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32802, 78, 65482, 32795, 32809, 20007, 32755, 45416] avg=32768.0000 max=65482 max_rank=2 min=78 min_rank=1 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] avg=32768.0000 max=33096 max_rank=2 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=44706 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32802, 78, 65482, 32795, 32809, 20007, 32755, 45416] avg=32768.0000 max=65482 max_rank=2 min=78 min_rank=1 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] avg=32768.0000 max=33096 max_rank=0 min=31796 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33767 max_load_reduction_pct=0.505018 moved_to_replicas=65796 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65, 37036, 32728, 32746, 32779, 32720, 82247, 11823] avg=32768.0000 max=82247 max_rank=6 min=65 min_rank=0 imbalance_ratio=2.509979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] avg=32768.0000 max=33096 max_rank=3 min=32106 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97460 max_load_reduction_pct=0.746500 moved_to_replicas=161797 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[23, 90, 20, 478, 130556, 32881, 97433, 663] avg=32768.0000 max=130556 max_rank=4 min=20 min_rank=2 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] avg=32768.0000 max=33096 max_rank=2 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=44706 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] avg=32768.0000 max=33096 max_rank=2 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=44706 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[44370, 32761, 32829, 32808, 3, 76401, 9049, 33923] avg=32768.0000 max=76401 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.331573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[23, 90, 20, 478, 130556, 32881, 97433, 663] avg=32768.0000 max=130556 max_rank=4 min=20 min_rank=2 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] avg=32768.0000 max=33096 max_rank=1 min=32209 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=49151 max_load_reduction_pct=0.597602 moved_to_replicas=53091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32802, 78, 65482, 32795, 32809, 20007, 32755, 45416] avg=32768.0000 max=65482 max_rank=2 min=78 min_rank=1 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] avg=32768.0000 max=33096 max_rank=3 min=32106 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97460 max_load_reduction_pct=0.746500 moved_to_replicas=161797 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65, 37036, 32728, 32746, 32779, 32720, 82247, 11823] avg=32768.0000 max=82247 max_rank=6 min=65 min_rank=0 imbalance_ratio=2.509979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[33095, 32761, 32829, 32808, 32728, 33097, 31730, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43304 max_load_reduction_pct=0.566799 moved_to_replicas=55406 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[23, 90, 20, 478, 130556, 32881, 97433, 663] avg=32768.0000 max=130556 max_rank=4 min=20 min_rank=2 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] avg=32768.0000 max=33096 max_rank=3 min=32106 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97460 max_load_reduction_pct=0.746500 moved_to_replicas=161797 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[23, 90, 20, 478, 130556, 32881, 97433, 663] avg=32768.0000 max=130556 max_rank=4 min=20 min_rank=2 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] avg=32768.0000 max=33096 max_rank=2 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=44706 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[83, 32782, 65536, 65460, 32403, 32757, 32702, 421] avg=32768.0000 max=65536 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] avg=32768.0000 max=33096 max_rank=1 min=32209 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=49151 max_load_reduction_pct=0.597602 moved_to_replicas=53091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] avg=32768.0000 max=33096 max_rank=3 min=32106 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97460 max_load_reduction_pct=0.746500 moved_to_replicas=161797 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65, 37036, 32728, 32746, 32779, 32720, 82247, 11823] avg=32768.0000 max=82247 max_rank=6 min=65 min_rank=0 imbalance_ratio=2.509979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] avg=32768.0000 max=33096 max_rank=3 min=32106 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97460 max_load_reduction_pct=0.746500 moved_to_replicas=161797 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] avg=32768.0000 max=33096 max_rank=2 min=32403 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64804 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[66863, 32837, 31759, 32762, 14, 65125, 32730, 54] avg=32768.0000 max=66863 max_rank=0 min=14 min_rank=4 imbalance_ratio=2.040497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65, 37036, 32728, 32746, 32779, 32720, 82247, 11823] avg=32768.0000 max=82247 max_rank=6 min=65 min_rank=0 imbalance_ratio=2.509979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[23, 90, 20, 478, 130556, 32881, 97433, 663] avg=32768.0000 max=130556 max_rank=4 min=20 min_rank=2 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] avg=32768.0000 max=33096 max_rank=1 min=32209 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=49151 max_load_reduction_pct=0.597602 moved_to_replicas=53091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[83, 32782, 65536, 65460, 32403, 32757, 32702, 421] avg=32768.0000 max=65536 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65, 37036, 32728, 32746, 32779, 32720, 82247, 11823] avg=32768.0000 max=82247 max_rank=6 min=65 min_rank=0 imbalance_ratio=2.509979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[33096, 32837, 31796, 32762, 32731, 33096, 32730, 33096] avg=32768.0000 max=33096 max_rank=0 min=31796 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=33767 max_load_reduction_pct=0.505018 moved_to_replicas=65796 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] avg=32768.0000 max=33096 max_rank=1 min=32209 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=49151 max_load_reduction_pct=0.597602 moved_to_replicas=53091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65, 37036, 32728, 32746, 32779, 32720, 82247, 11823] avg=32768.0000 max=82247 max_rank=6 min=65 min_rank=0 imbalance_ratio=2.509979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] avg=32768.0000 max=33096 max_rank=3 min=32106 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97460 max_load_reduction_pct=0.746500 moved_to_replicas=161797 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[8, 32841, 2209, 65474, 32799, 87004, 9040, 32769] avg=32768.0000 max=87004 max_rank=5 min=8 min_rank=0 imbalance_ratio=2.655151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] avg=32768.0000 max=33096 max_rank=2 min=32403 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64804 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] avg=32768.0000 max=33096 max_rank=1 min=32209 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=49151 max_load_reduction_pct=0.597602 moved_to_replicas=53091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[83, 32782, 65536, 65460, 32403, 32757, 32702, 421] avg=32768.0000 max=65536 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] avg=32768.0000 max=33096 max_rank=1 min=32209 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=49151 max_load_reduction_pct=0.597602 moved_to_replicas=53091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] avg=32768.0000 max=33096 max_rank=3 min=31716 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=53908 max_load_reduction_pct=0.619604 moved_to_replicas=86286 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[83, 32782, 65536, 65460, 32403, 32757, 32702, 421] avg=32768.0000 max=65536 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32802, 78, 65482, 32795, 32809, 20007, 32755, 45416] avg=32768.0000 max=65482 max_rank=2 min=78 min_rank=1 imbalance_ratio=1.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65, 37036, 32728, 32746, 32779, 32720, 82247, 11823] avg=32768.0000 max=82247 max_rank=6 min=65 min_rank=0 imbalance_ratio=2.509979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] avg=32768.0000 max=33096 max_rank=2 min=32403 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64804 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[8, 32841, 2209, 65474, 32799, 87004, 9040, 32769] avg=32768.0000 max=87004 max_rank=5 min=8 min_rank=0 imbalance_ratio=2.655151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[83, 32782, 65536, 65460, 32403, 32757, 32702, 421] avg=32768.0000 max=65536 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] avg=32768.0000 max=33096 max_rank=2 min=32403 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64804 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[83, 32782, 65536, 65460, 32403, 32757, 32702, 421] avg=32768.0000 max=65536 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32802, 32464, 33096, 32795, 32809, 32327, 32755, 33096] avg=32768.0000 max=33096 max_rank=2 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=44706 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] avg=32768.0000 max=33096 max_rank=1 min=32209 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=49151 max_load_reduction_pct=0.597602 moved_to_replicas=53091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32773, 98126, 65382, 32808, 43, 125, 32872, 15] avg=32768.0000 max=98126 max_rank=1 min=15 min_rank=7 imbalance_ratio=2.994568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] avg=32768.0000 max=33096 max_rank=3 min=31716 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=53908 max_load_reduction_pct=0.619604 moved_to_replicas=86286 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] avg=32768.0000 max=33096 max_rank=2 min=32403 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64804 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[8, 32841, 2209, 65474, 32799, 87004, 9040, 32769] avg=32768.0000 max=87004 max_rank=5 min=8 min_rank=0 imbalance_ratio=2.655151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] avg=32768.0000 max=33096 max_rank=2 min=32403 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64804 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] avg=32768.0000 max=33096 max_rank=1 min=32356 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65030 max_load_reduction_pct=0.662719 moved_to_replicas=97316 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[8, 32841, 2209, 65474, 32799, 87004, 9040, 32769] avg=32768.0000 max=87004 max_rank=5 min=8 min_rank=0 imbalance_ratio=2.655151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[83, 32782, 65536, 65460, 32403, 32757, 32702, 421] avg=32768.0000 max=65536 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[23, 90, 20, 478, 130556, 32881, 97433, 663] avg=32768.0000 max=130556 max_rank=4 min=20 min_rank=2 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] avg=32768.0000 max=33096 max_rank=3 min=31716 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=53908 max_load_reduction_pct=0.619604 moved_to_replicas=86286 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32773, 98126, 65382, 32808, 43, 125, 32872, 15] avg=32768.0000 max=98126 max_rank=1 min=15 min_rank=7 imbalance_ratio=2.994568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[8, 32841, 2209, 65474, 32799, 87004, 9040, 32769] avg=32768.0000 max=87004 max_rank=5 min=8 min_rank=0 imbalance_ratio=2.655151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] avg=32768.0000 max=33096 max_rank=3 min=31716 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=53908 max_load_reduction_pct=0.619604 moved_to_replicas=86286 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[8, 32841, 2209, 65474, 32799, 87004, 9040, 32769] avg=32768.0000 max=87004 max_rank=5 min=8 min_rank=0 imbalance_ratio=2.655151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[101, 65489, 65378, 65486, 23, 32775, 32885, 7] avg=32768.0000 max=65489 max_rank=1 min=7 min_rank=7 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] avg=32768.0000 max=33096 max_rank=2 min=32403 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64804 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32743, 32106, 32744, 33096, 33096, 32881, 33096, 32382] avg=32768.0000 max=33096 max_rank=3 min=32106 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97460 max_load_reduction_pct=0.746500 moved_to_replicas=161797 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] avg=32768.0000 max=33096 max_rank=1 min=32356 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65030 max_load_reduction_pct=0.662719 moved_to_replicas=97316 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] avg=32768.0000 max=33096 max_rank=3 min=31716 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=53908 max_load_reduction_pct=0.619604 moved_to_replicas=86286 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32773, 98126, 65382, 32808, 43, 125, 32872, 15] avg=32768.0000 max=98126 max_rank=1 min=15 min_rank=7 imbalance_ratio=2.994568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] avg=32768.0000 max=33096 max_rank=3 min=31716 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=53908 max_load_reduction_pct=0.619604 moved_to_replicas=86286 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] avg=32768.0000 max=33096 max_rank=1 min=32383 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=97065 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32773, 98126, 65382, 32808, 43, 125, 32872, 15] avg=32768.0000 max=98126 max_rank=1 min=15 min_rank=7 imbalance_ratio=2.994568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] avg=32768.0000 max=33096 max_rank=1 min=32356 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65030 max_load_reduction_pct=0.662719 moved_to_replicas=97316 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65, 37036, 32728, 32746, 32779, 32720, 82247, 11823] avg=32768.0000 max=82247 max_rank=6 min=65 min_rank=0 imbalance_ratio=2.509979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[8, 32841, 2209, 65474, 32799, 87004, 9040, 32769] avg=32768.0000 max=87004 max_rank=5 min=8 min_rank=0 imbalance_ratio=2.655151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[101, 65489, 65378, 65486, 23, 32775, 32885, 7] avg=32768.0000 max=65489 max_rank=1 min=7 min_rank=7 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32773, 98126, 65382, 32808, 43, 125, 32872, 15] avg=32768.0000 max=98126 max_rank=1 min=15 min_rank=7 imbalance_ratio=2.994568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32773, 98126, 65382, 32808, 43, 125, 32872, 15] avg=32768.0000 max=98126 max_rank=1 min=15 min_rank=7 imbalance_ratio=2.994568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] avg=32768.0000 max=33096 max_rank=1 min=32356 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65030 max_load_reduction_pct=0.662719 moved_to_replicas=97316 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[46756, 32758, 32786, 32430, 65769, 18922, 52, 32671] avg=32768.0000 max=65769 max_rank=4 min=52 min_rank=6 imbalance_ratio=2.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] avg=32768.0000 max=33096 max_rank=3 min=31716 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=53908 max_load_reduction_pct=0.619604 moved_to_replicas=86286 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32771, 33096, 32728, 32746, 32779, 32720, 33095, 32209] avg=32768.0000 max=33096 max_rank=1 min=32209 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=49151 max_load_reduction_pct=0.597602 moved_to_replicas=53091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] avg=32768.0000 max=33096 max_rank=1 min=32383 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=97065 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] avg=32768.0000 max=33096 max_rank=1 min=32356 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65030 max_load_reduction_pct=0.662719 moved_to_replicas=97316 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[101, 65489, 65378, 65486, 23, 32775, 32885, 7] avg=32768.0000 max=65489 max_rank=1 min=7 min_rank=7 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] avg=32768.0000 max=33096 max_rank=1 min=32356 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65030 max_load_reduction_pct=0.662719 moved_to_replicas=97316 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] avg=32768.0000 max=33096 max_rank=4 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32673 max_load_reduction_pct=0.496784 moved_to_replicas=46333 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[101, 65489, 65378, 65486, 23, 32775, 32885, 7] avg=32768.0000 max=65489 max_rank=1 min=7 min_rank=7 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] avg=32768.0000 max=33096 max_rank=1 min=32383 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=97065 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32773, 98126, 65382, 32808, 43, 125, 32872, 15] avg=32768.0000 max=98126 max_rank=1 min=15 min_rank=7 imbalance_ratio=2.994568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[46756, 32758, 32786, 32430, 65769, 18922, 52, 32671] avg=32768.0000 max=65769 max_rank=4 min=52 min_rank=6 imbalance_ratio=2.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[101, 65489, 65378, 65486, 23, 32775, 32885, 7] avg=32768.0000 max=65489 max_rank=1 min=7 min_rank=7 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[83, 32782, 65536, 65460, 32403, 32757, 32702, 421] avg=32768.0000 max=65536 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] avg=32768.0000 max=33096 max_rank=1 min=32383 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=97065 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[101, 65489, 65378, 65486, 23, 32775, 32885, 7] avg=32768.0000 max=65489 max_rank=1 min=7 min_rank=7 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[32815, 123, 32733, 32764, 65474, 415, 65029, 32791] avg=32768.0000 max=65474 max_rank=4 min=123 min_rank=1 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] avg=32768.0000 max=33096 max_rank=1 min=32356 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65030 max_load_reduction_pct=0.662719 moved_to_replicas=97316 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] avg=32768.0000 max=33096 max_rank=4 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32673 max_load_reduction_pct=0.496784 moved_to_replicas=46333 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] avg=32768.0000 max=33096 max_rank=1 min=32383 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=97065 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32523, 32782, 33096, 33096, 32403, 32757, 32702, 32785] avg=32768.0000 max=33096 max_rank=2 min=32403 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64804 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] avg=32768.0000 max=33096 max_rank=1 min=32383 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=97065 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[46756, 32758, 32786, 32430, 65769, 18922, 52, 32671] avg=32768.0000 max=65769 max_rank=4 min=52 min_rank=6 imbalance_ratio=2.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] avg=32768.0000 max=33096 max_rank=4 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64311 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[46756, 32758, 32786, 32430, 65769, 18922, 52, 32671] avg=32768.0000 max=65769 max_rank=4 min=52 min_rank=6 imbalance_ratio=2.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] avg=32768.0000 max=33096 max_rank=4 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32673 max_load_reduction_pct=0.496784 moved_to_replicas=46333 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[101, 65489, 65378, 65486, 23, 32775, 32885, 7] avg=32768.0000 max=65489 max_rank=1 min=7 min_rank=7 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[32815, 123, 32733, 32764, 65474, 415, 65029, 32791] avg=32768.0000 max=65474 max_rank=4 min=123 min_rank=1 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[46756, 32758, 32786, 32430, 65769, 18922, 52, 32671] avg=32768.0000 max=65769 max_rank=4 min=52 min_rank=6 imbalance_ratio=2.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] avg=32768.0000 max=33096 max_rank=4 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32673 max_load_reduction_pct=0.496784 moved_to_replicas=46333 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[46756, 32758, 32786, 32430, 65769, 18922, 52, 32671] avg=32768.0000 max=65769 max_rank=4 min=52 min_rank=6 imbalance_ratio=2.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[8, 32841, 2209, 65474, 32799, 87004, 9040, 32769] avg=32768.0000 max=87004 max_rank=5 min=8 min_rank=0 imbalance_ratio=2.655151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[12, 47, 51, 32776, 65544, 67223, 32762, 63729] avg=32768.0000 max=67223 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.051483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] avg=32768.0000 max=33096 max_rank=1 min=32383 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=97065 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] avg=32768.0000 max=33096 max_rank=4 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64311 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] avg=32768.0000 max=33096 max_rank=4 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32673 max_load_reduction_pct=0.496784 moved_to_replicas=46333 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[32815, 123, 32733, 32764, 65474, 415, 65029, 32791] avg=32768.0000 max=65474 max_rank=4 min=123 min_rank=1 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] avg=32768.0000 max=33096 max_rank=4 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32673 max_load_reduction_pct=0.496784 moved_to_replicas=46333 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32732, 32841, 31716, 33096, 32799, 33096, 33095, 32769] avg=32768.0000 max=33096 max_rank=3 min=31716 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=53908 max_load_reduction_pct=0.619604 moved_to_replicas=86286 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] avg=32768.0000 max=33096 max_rank=4 min=32087 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=34127 max_load_reduction_pct=0.507669 moved_to_replicas=97208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[32815, 123, 32733, 32764, 65474, 415, 65029, 32791] avg=32768.0000 max=65474 max_rank=4 min=123 min_rank=1 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] avg=32768.0000 max=33096 max_rank=4 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64311 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[46756, 32758, 32786, 32430, 65769, 18922, 52, 32671] avg=32768.0000 max=65769 max_rank=4 min=52 min_rank=6 imbalance_ratio=2.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] avg=32768.0000 max=33096 max_rank=4 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64311 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[12, 47, 51, 32776, 65544, 67223, 32762, 63729] avg=32768.0000 max=67223 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.051483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[32815, 123, 32733, 32764, 65474, 415, 65029, 32791] avg=32768.0000 max=65474 max_rank=4 min=123 min_rank=1 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[32815, 123, 32733, 32764, 65474, 415, 65029, 32791] avg=32768.0000 max=65474 max_rank=4 min=123 min_rank=1 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 42, 32810, 41, 65461, 32824, 65448] avg=32768.0000 max=65461 max_rank=5 min=41 min_rank=4 imbalance_ratio=1.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32773, 98126, 65382, 32808, 43, 125, 32872, 15] avg=32768.0000 max=98126 max_rank=1 min=15 min_rank=7 imbalance_ratio=2.994568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] avg=32768.0000 max=33096 max_rank=4 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32673 max_load_reduction_pct=0.496784 moved_to_replicas=46333 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[12, 47, 51, 32776, 65544, 67223, 32762, 63729] avg=32768.0000 max=67223 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.051483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] avg=32768.0000 max=33096 max_rank=4 min=32087 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=34127 max_load_reduction_pct=0.507669 moved_to_replicas=97208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] avg=32768.0000 max=33096 max_rank=4 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64311 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] avg=32768.0000 max=33096 max_rank=4 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64311 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32394 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32365 max_load_reduction_pct=0.494417 moved_to_replicas=64717 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32773, 33096, 33096, 32808, 32356, 32411, 32872, 32732] avg=32768.0000 max=33096 max_rank=1 min=32356 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65030 max_load_reduction_pct=0.662719 moved_to_replicas=97316 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[12, 47, 51, 32776, 65544, 67223, 32762, 63729] avg=32768.0000 max=67223 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.051483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] avg=32768.0000 max=33096 max_rank=4 min=32087 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=34127 max_load_reduction_pct=0.507669 moved_to_replicas=97208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] avg=32768.0000 max=33096 max_rank=4 min=32087 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=34127 max_load_reduction_pct=0.507669 moved_to_replicas=97208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[32815, 123, 32733, 32764, 65474, 415, 65029, 32791] avg=32768.0000 max=65474 max_rank=4 min=123 min_rank=1 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 42, 32810, 41, 65461, 32824, 65448] avg=32768.0000 max=65461 max_rank=5 min=41 min_rank=4 imbalance_ratio=1.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[12, 47, 51, 32776, 65544, 67223, 32762, 63729] avg=32768.0000 max=67223 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.051483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[12, 47, 51, 32776, 65544, 67223, 32762, 63729] avg=32768.0000 max=67223 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.051483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[65119, 219, 32777, 174, 100, 32774, 97955, 33026] avg=32768.0000 max=97955 max_rank=6 min=100 min_rank=4 imbalance_ratio=2.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[101, 65489, 65378, 65486, 23, 32775, 32885, 7] avg=32768.0000 max=65489 max_rank=1 min=7 min_rank=7 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 42, 32810, 41, 65461, 32824, 65448] avg=32768.0000 max=65461 max_rank=5 min=41 min_rank=4 imbalance_ratio=1.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32394 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32365 max_load_reduction_pct=0.494417 moved_to_replicas=64717 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] avg=32768.0000 max=33096 max_rank=4 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64311 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] avg=32768.0000 max=33096 max_rank=4 min=32087 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=34127 max_load_reduction_pct=0.507669 moved_to_replicas=97208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] avg=32768.0000 max=33096 max_rank=4 min=32087 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=34127 max_load_reduction_pct=0.507669 moved_to_replicas=97208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] avg=32768.0000 max=33096 max_rank=0 min=32242 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.662131 moved_to_replicas=96882 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 42, 32810, 41, 65461, 32824, 65448] avg=32768.0000 max=65461 max_rank=5 min=41 min_rank=4 imbalance_ratio=1.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32383, 33096, 33096, 33096, 32413, 32775, 32885, 32400] avg=32768.0000 max=33096 max_rank=1 min=32383 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=97065 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32394 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32365 max_load_reduction_pct=0.494417 moved_to_replicas=64717 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[65119, 219, 32777, 174, 100, 32774, 97955, 33026] avg=32768.0000 max=97955 max_rank=6 min=100 min_rank=4 imbalance_ratio=2.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32394 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32365 max_load_reduction_pct=0.494417 moved_to_replicas=64717 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 42, 32810, 41, 65461, 32824, 65448] avg=32768.0000 max=65461 max_rank=5 min=41 min_rank=4 imbalance_ratio=1.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[12, 47, 51, 32776, 65544, 67223, 32762, 63729] avg=32768.0000 max=67223 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.051483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 42, 32810, 41, 65461, 32824, 65448] avg=32768.0000 max=65461 max_rank=5 min=41 min_rank=4 imbalance_ratio=1.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[32805, 36, 32774, 82522, 74, 48618, 32564, 32751] avg=32768.0000 max=82522 max_rank=3 min=36 min_rank=1 imbalance_ratio=2.518372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[46756, 32758, 32786, 32430, 65769, 18922, 52, 32671] avg=32768.0000 max=65769 max_rank=4 min=52 min_rank=6 imbalance_ratio=2.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] avg=32768.0000 max=33096 max_rank=0 min=32242 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.662131 moved_to_replicas=96882 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[65119, 219, 32777, 174, 100, 32774, 97955, 33026] avg=32768.0000 max=97955 max_rank=6 min=100 min_rank=4 imbalance_ratio=2.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32394 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32365 max_load_reduction_pct=0.494417 moved_to_replicas=64717 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] avg=32768.0000 max=33096 max_rank=4 min=32087 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=34127 max_load_reduction_pct=0.507669 moved_to_replicas=97208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32394 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32365 max_load_reduction_pct=0.494417 moved_to_replicas=64717 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] avg=32768.0000 max=33097 max_rank=3 min=32300 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49425 max_load_reduction_pct=0.598931 moved_to_replicas=64948 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[65119, 219, 32777, 174, 100, 32774, 97955, 33026] avg=32768.0000 max=97955 max_rank=6 min=100 min_rank=4 imbalance_ratio=2.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[33095, 32758, 32786, 32430, 33096, 32583, 32725, 32671] avg=32768.0000 max=33096 max_rank=4 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32673 max_load_reduction_pct=0.496784 moved_to_replicas=46333 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] avg=32768.0000 max=33096 max_rank=0 min=32242 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.662131 moved_to_replicas=96882 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[32805, 36, 32774, 82522, 74, 48618, 32564, 32751] avg=32768.0000 max=82522 max_rank=3 min=36 min_rank=1 imbalance_ratio=2.518372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] avg=32768.0000 max=33096 max_rank=0 min=32242 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.662131 moved_to_replicas=96882 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[65119, 219, 32777, 174, 100, 32774, 97955, 33026] avg=32768.0000 max=97955 max_rank=6 min=100 min_rank=4 imbalance_ratio=2.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 42, 32810, 41, 65461, 32824, 65448] avg=32768.0000 max=65461 max_rank=5 min=41 min_rank=4 imbalance_ratio=1.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[65119, 219, 32777, 174, 100, 32774, 97955, 33026] avg=32768.0000 max=97955 max_rank=6 min=100 min_rank=4 imbalance_ratio=2.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[34918, 63454, 32622, 142, 83, 32810, 98023, 92] avg=32768.0000 max=98023 max_rank=6 min=83 min_rank=4 imbalance_ratio=2.991425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[32815, 123, 32733, 32764, 65474, 415, 65029, 32791] avg=32768.0000 max=65474 max_rank=4 min=123 min_rank=1 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] avg=32768.0000 max=33097 max_rank=3 min=32300 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49425 max_load_reduction_pct=0.598931 moved_to_replicas=64948 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[32805, 36, 32774, 82522, 74, 48618, 32564, 32751] avg=32768.0000 max=82522 max_rank=3 min=36 min_rank=1 imbalance_ratio=2.518372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] avg=32768.0000 max=33096 max_rank=0 min=32242 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.662131 moved_to_replicas=96882 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32394 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32365 max_load_reduction_pct=0.494417 moved_to_replicas=64717 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] avg=32768.0000 max=33096 max_rank=0 min=32242 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.662131 moved_to_replicas=96882 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] avg=32768.0000 max=33096 max_rank=0 min=32298 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64927 max_load_reduction_pct=0.662365 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[32805, 36, 32774, 82522, 74, 48618, 32564, 32751] avg=32768.0000 max=82522 max_rank=3 min=36 min_rank=1 imbalance_ratio=2.518372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[32815, 32501, 32733, 32764, 33096, 32348, 33096, 32791] avg=32768.0000 max=33096 max_rank=4 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64311 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] avg=32768.0000 max=33097 max_rank=3 min=32300 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49425 max_load_reduction_pct=0.598931 moved_to_replicas=64948 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[34918, 63454, 32622, 142, 83, 32810, 98023, 92] avg=32768.0000 max=98023 max_rank=6 min=83 min_rank=4 imbalance_ratio=2.991425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[32805, 36, 32774, 82522, 74, 48618, 32564, 32751] avg=32768.0000 max=82522 max_rank=3 min=36 min_rank=1 imbalance_ratio=2.518372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] avg=32768.0000 max=33097 max_rank=3 min=32300 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49425 max_load_reduction_pct=0.598931 moved_to_replicas=64948 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[65119, 219, 32777, 174, 100, 32774, 97955, 33026] avg=32768.0000 max=97955 max_rank=6 min=100 min_rank=4 imbalance_ratio=2.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[32805, 36, 32774, 82522, 74, 48618, 32564, 32751] avg=32768.0000 max=82522 max_rank=3 min=36 min_rank=1 imbalance_ratio=2.518372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[32740, 72, 32768, 98224, 32736, 321, 29, 65254] avg=32768.0000 max=98224 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] avg=32768.0000 max=33096 max_rank=0 min=32298 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64927 max_load_reduction_pct=0.662365 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[34918, 63454, 32622, 142, 83, 32810, 98023, 92] avg=32768.0000 max=98023 max_rank=6 min=83 min_rank=4 imbalance_ratio=2.991425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[12, 47, 51, 32776, 65544, 67223, 32762, 63729] avg=32768.0000 max=67223 max_rank=5 min=12 min_rank=0 imbalance_ratio=2.051483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] avg=32768.0000 max=33097 max_rank=3 min=32300 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49425 max_load_reduction_pct=0.598931 moved_to_replicas=64948 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] avg=32768.0000 max=33096 max_rank=0 min=32242 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.662131 moved_to_replicas=96882 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] avg=32768.0000 max=33097 max_rank=3 min=32300 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49425 max_load_reduction_pct=0.598931 moved_to_replicas=64948 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] avg=32768.0000 max=33096 max_rank=3 min=32477 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97286 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[34918, 63454, 32622, 142, 83, 32810, 98023, 92] avg=32768.0000 max=98023 max_rank=6 min=83 min_rank=4 imbalance_ratio=2.991425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] avg=32768.0000 max=33096 max_rank=0 min=32298 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64927 max_load_reduction_pct=0.662365 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[32732, 32087, 32499, 32776, 33096, 33096, 32762, 33096] avg=32768.0000 max=33096 max_rank=4 min=32087 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=34127 max_load_reduction_pct=0.507669 moved_to_replicas=97208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[32740, 72, 32768, 98224, 32736, 321, 29, 65254] avg=32768.0000 max=98224 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[34918, 63454, 32622, 142, 83, 32810, 98023, 92] avg=32768.0000 max=98023 max_rank=6 min=83 min_rank=4 imbalance_ratio=2.991425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] avg=32768.0000 max=33096 max_rank=0 min=32298 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64927 max_load_reduction_pct=0.662365 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[32805, 36, 32774, 82522, 74, 48618, 32564, 32751] avg=32768.0000 max=82522 max_rank=3 min=36 min_rank=1 imbalance_ratio=2.518372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[112, 32724, 65569, 24, 32715, 32721, 98203, 76] avg=32768.0000 max=98203 max_rank=6 min=24 min_rank=3 imbalance_ratio=2.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[34918, 63454, 32622, 142, 83, 32810, 98023, 92] avg=32768.0000 max=98023 max_rank=6 min=83 min_rank=4 imbalance_ratio=2.991425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] avg=32768.0000 max=33096 max_rank=3 min=32477 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97286 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[32740, 72, 32768, 98224, 32736, 321, 29, 65254] avg=32768.0000 max=98224 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 42, 32810, 41, 65461, 32824, 65448] avg=32768.0000 max=65461 max_rank=5 min=41 min_rank=4 imbalance_ratio=1.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] avg=32768.0000 max=33096 max_rank=0 min=32298 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64927 max_load_reduction_pct=0.662365 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] avg=32768.0000 max=33097 max_rank=3 min=32300 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49425 max_load_reduction_pct=0.598931 moved_to_replicas=64948 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65107 max_load_reduction_pct=0.662984 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] avg=32768.0000 max=33096 max_rank=0 min=32298 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64927 max_load_reduction_pct=0.662365 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[32740, 72, 32768, 98224, 32736, 321, 29, 65254] avg=32768.0000 max=98224 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] avg=32768.0000 max=33096 max_rank=3 min=32477 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97286 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[32738, 32780, 32394, 32810, 32406, 33096, 32824, 33096] avg=32768.0000 max=33096 max_rank=5 min=32394 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32365 max_load_reduction_pct=0.494417 moved_to_replicas=64717 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[112, 32724, 65569, 24, 32715, 32721, 98203, 76] avg=32768.0000 max=98203 max_rank=6 min=24 min_rank=3 imbalance_ratio=2.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[32740, 72, 32768, 98224, 32736, 321, 29, 65254] avg=32768.0000 max=98224 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] avg=32768.0000 max=33096 max_rank=3 min=32477 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97286 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[34918, 63454, 32622, 142, 83, 32810, 98023, 92] avg=32768.0000 max=98023 max_rank=6 min=83 min_rank=4 imbalance_ratio=2.991425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=25 tokens_per_ep_rank=[97, 73, 32753, 98177, 26729, 38855, 32686, 32774] avg=32768.0000 max=98177 max_rank=3 min=73 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65107 max_load_reduction_pct=0.662984 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[32740, 72, 32768, 98224, 32736, 321, 29, 65254] avg=32768.0000 max=98224 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[112, 32724, 65569, 24, 32715, 32721, 98203, 76] avg=32768.0000 max=98203 max_rank=6 min=24 min_rank=3 imbalance_ratio=2.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] avg=32768.0000 max=33096 max_rank=3 min=32477 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97286 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[65119, 219, 32777, 174, 100, 32774, 97955, 33026] avg=32768.0000 max=97955 max_rank=6 min=100 min_rank=4 imbalance_ratio=2.989349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] avg=32768.0000 max=33096 max_rank=0 min=32298 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64927 max_load_reduction_pct=0.662365 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[112, 32724, 65569, 24, 32715, 32721, 98203, 76] avg=32768.0000 max=98203 max_rank=6 min=24 min_rank=3 imbalance_ratio=2.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=25 tokens_per_ep_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] avg=32768.0000 max=33096 max_rank=3 min=32455 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70840 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] avg=32768.0000 max=33096 max_rank=3 min=32477 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97286 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65107 max_load_reduction_pct=0.662984 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[33096, 32242, 32777, 32310, 32823, 32774, 33096, 33026] avg=32768.0000 max=33096 max_rank=0 min=32242 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.662131 moved_to_replicas=96882 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=25 tokens_per_ep_rank=[97, 73, 32753, 98177, 26729, 38855, 32686, 32774] avg=32768.0000 max=98177 max_rank=3 min=73 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[112, 32724, 65569, 24, 32715, 32721, 98203, 76] avg=32768.0000 max=98203 max_rank=6 min=24 min_rank=3 imbalance_ratio=2.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65107 max_load_reduction_pct=0.662984 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[32740, 72, 32768, 98224, 32736, 321, 29, 65254] avg=32768.0000 max=98224 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=26 tokens_per_ep_rank=[32794, 30, 33614, 46, 64825, 32701, 65392, 32742] avg=32768.0000 max=65392 max_rank=6 min=30 min_rank=1 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[112, 32724, 65569, 24, 32715, 32721, 98203, 76] avg=32768.0000 max=98203 max_rank=6 min=24 min_rank=3 imbalance_ratio=2.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=25 tokens_per_ep_rank=[97, 73, 32753, 98177, 26729, 38855, 32686, 32774] avg=32768.0000 max=98177 max_rank=3 min=73 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65107 max_load_reduction_pct=0.662984 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=25 tokens_per_ep_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] avg=32768.0000 max=33096 max_rank=3 min=32455 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70840 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[32805, 36, 32774, 82522, 74, 48618, 32564, 32751] avg=32768.0000 max=82522 max_rank=3 min=36 min_rank=1 imbalance_ratio=2.518372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] avg=32768.0000 max=33096 max_rank=3 min=32477 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97286 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=25 tokens_per_ep_rank=[97, 73, 32753, 98177, 26729, 38855, 32686, 32774] avg=32768.0000 max=98177 max_rank=3 min=73 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=26 tokens_per_ep_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32293 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=64543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65107 max_load_reduction_pct=0.662984 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=25 tokens_per_ep_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] avg=32768.0000 max=33096 max_rank=3 min=32455 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70840 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[32805, 32758, 32774, 33097, 32300, 33095, 32564, 32751] avg=32768.0000 max=33097 max_rank=3 min=32300 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49425 max_load_reduction_pct=0.598931 moved_to_replicas=64948 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=26 tokens_per_ep_rank=[32794, 30, 33614, 46, 64825, 32701, 65392, 32742] avg=32768.0000 max=65392 max_rank=6 min=30 min_rank=1 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=25 tokens_per_ep_rank=[97, 73, 32753, 98177, 26729, 38855, 32686, 32774] avg=32768.0000 max=98177 max_rank=3 min=73 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=25 tokens_per_ep_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] avg=32768.0000 max=33096 max_rank=3 min=32455 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70840 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[112, 32724, 65569, 24, 32715, 32721, 98203, 76] avg=32768.0000 max=98203 max_rank=6 min=24 min_rank=3 imbalance_ratio=2.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 79, 32862, 32799, 65486, 32741] avg=32768.0000 max=65486 max_rank=6 min=79 min_rank=3 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=26 tokens_per_ep_rank=[32794, 30, 33614, 46, 64825, 32701, 65392, 32742] avg=32768.0000 max=65392 max_rank=6 min=30 min_rank=1 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=26 tokens_per_ep_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32293 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=64543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=25 tokens_per_ep_rank=[97, 73, 32753, 98177, 26729, 38855, 32686, 32774] avg=32768.0000 max=98177 max_rank=3 min=73 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=25 tokens_per_ep_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] avg=32768.0000 max=33096 max_rank=3 min=32455 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70840 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[34918, 63454, 32622, 142, 83, 32810, 98023, 92] avg=32768.0000 max=98023 max_rank=6 min=83 min_rank=4 imbalance_ratio=2.991425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65107 max_load_reduction_pct=0.662984 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=32390 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=26 tokens_per_ep_rank=[32794, 30, 33614, 46, 64825, 32701, 65392, 32742] avg=32768.0000 max=65392 max_rank=6 min=30 min_rank=1 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=26 tokens_per_ep_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32293 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=64543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=25 tokens_per_ep_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] avg=32768.0000 max=33096 max_rank=3 min=32455 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70840 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[33096, 33096, 32622, 32322, 32804, 32810, 33096, 32298] avg=32768.0000 max=33096 max_rank=0 min=32298 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64927 max_load_reduction_pct=0.662365 moved_to_replicas=97107 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 79, 32862, 32799, 65486, 32741] avg=32768.0000 max=65486 max_rank=6 min=79 min_rank=3 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=26 tokens_per_ep_rank=[32794, 30, 33614, 46, 64825, 32701, 65392, 32742] avg=32768.0000 max=65392 max_rank=6 min=30 min_rank=1 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=26 tokens_per_ep_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32293 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=64543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=25 tokens_per_ep_rank=[97, 73, 32753, 98177, 26729, 38855, 32686, 32774] avg=32768.0000 max=98177 max_rank=3 min=73 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=28 tokens_per_ep_rank=[32701, 65453, 54, 32761, 65480, 86, 89, 65520] avg=32768.0000 max=65520 max_rank=7 min=54 min_rank=2 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 79, 32862, 32799, 65486, 32741] avg=32768.0000 max=65486 max_rank=6 min=79 min_rank=3 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=32390 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=26 tokens_per_ep_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32293 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=64543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=26 tokens_per_ep_rank=[32794, 30, 33614, 46, 64825, 32701, 65392, 32742] avg=32768.0000 max=65392 max_rank=6 min=30 min_rank=1 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[32740, 72, 32768, 98224, 32736, 321, 29, 65254] avg=32768.0000 max=98224 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=25 tokens_per_ep_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] avg=32768.0000 max=33096 max_rank=3 min=32455 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70840 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=28 tokens_per_ep_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] avg=32768.0000 max=33096 max_rank=1 min=32446 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=97165 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 79, 32862, 32799, 65486, 32741] avg=32768.0000 max=65486 max_rank=6 min=79 min_rank=3 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=32390 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=26 tokens_per_ep_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32293 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=64543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[32740, 32477, 32768, 33096, 32736, 32479, 32752, 33096] avg=32768.0000 max=33096 max_rank=3 min=32477 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65128 max_load_reduction_pct=0.663056 moved_to_replicas=97286 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=28 tokens_per_ep_rank=[32701, 65453, 54, 32761, 65480, 86, 89, 65520] avg=32768.0000 max=65520 max_rank=7 min=54 min_rank=2 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 79, 32862, 32799, 65486, 32741] avg=32768.0000 max=65486 max_rank=6 min=79 min_rank=3 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=32390 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=26 tokens_per_ep_rank=[32794, 30, 33614, 46, 64825, 32701, 65392, 32742] avg=32768.0000 max=65392 max_rank=6 min=30 min_rank=1 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=29 tokens_per_ep_rank=[113, 65513, 65500, 32725, 65480, 15, 32785, 13] avg=32768.0000 max=65513 max_rank=1 min=13 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=28 tokens_per_ep_rank=[32701, 65453, 54, 32761, 65480, 86, 89, 65520] avg=32768.0000 max=65520 max_rank=7 min=54 min_rank=2 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=28 tokens_per_ep_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] avg=32768.0000 max=33096 max_rank=1 min=32446 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=97165 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=32390 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 79, 32862, 32799, 65486, 32741] avg=32768.0000 max=65486 max_rank=6 min=79 min_rank=3 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[112, 32724, 65569, 24, 32715, 32721, 98203, 76] avg=32768.0000 max=98203 max_rank=6 min=24 min_rank=3 imbalance_ratio=2.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=26 tokens_per_ep_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32293 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=64543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=29 tokens_per_ep_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] avg=32768.0000 max=33096 max_rank=1 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=97205 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=28 tokens_per_ep_rank=[32701, 65453, 54, 32761, 65480, 86, 89, 65520] avg=32768.0000 max=65520 max_rank=7 min=54 min_rank=2 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=28 tokens_per_ep_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] avg=32768.0000 max=33096 max_rank=1 min=32446 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=97165 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=32390 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=29 tokens_per_ep_rank=[113, 65513, 65500, 32725, 65480, 15, 32785, 13] avg=32768.0000 max=65513 max_rank=1 min=13 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=28 tokens_per_ep_rank=[32701, 65453, 54, 32761, 65480, 86, 89, 65520] avg=32768.0000 max=65520 max_rank=7 min=54 min_rank=2 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[32585, 32724, 33096, 32731, 32715, 32721, 33096, 32476] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65107 max_load_reduction_pct=0.662984 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=28 tokens_per_ep_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] avg=32768.0000 max=33096 max_rank=1 min=32446 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=97165 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=30 tokens_per_ep_rank=[46, 65443, 32722, 32799, 32801, 65480, 32726, 127] avg=32768.0000 max=65480 max_rank=5 min=46 min_rank=0 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 79, 32862, 32799, 65486, 32741] avg=32768.0000 max=65486 max_rank=6 min=79 min_rank=3 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=29 tokens_per_ep_rank=[113, 65513, 65500, 32725, 65480, 15, 32785, 13] avg=32768.0000 max=65513 max_rank=1 min=13 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=29 tokens_per_ep_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] avg=32768.0000 max=33096 max_rank=1 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=97205 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=28 tokens_per_ep_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] avg=32768.0000 max=33096 max_rank=1 min=32446 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=97165 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=28 tokens_per_ep_rank=[32701, 65453, 54, 32761, 65480, 86, 89, 65520] avg=32768.0000 max=65520 max_rank=7 min=54 min_rank=2 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=64731 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=32390 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=25 tokens_per_ep_rank=[97, 73, 32753, 98177, 26729, 38855, 32686, 32774] avg=32768.0000 max=98177 max_rank=3 min=73 min_rank=1 imbalance_ratio=2.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=29 tokens_per_ep_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] avg=32768.0000 max=33096 max_rank=1 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=97205 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=29 tokens_per_ep_rank=[113, 65513, 65500, 32725, 65480, 15, 32785, 13] avg=32768.0000 max=65513 max_rank=1 min=13 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=28 tokens_per_ep_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] avg=32768.0000 max=33096 max_rank=1 min=32446 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=97165 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=30 tokens_per_ep_rank=[46, 65443, 32722, 32799, 32801, 65480, 32726, 127] avg=32768.0000 max=65480 max_rank=5 min=46 min_rank=0 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=29 tokens_per_ep_rank=[113, 65513, 65500, 32725, 65480, 15, 32785, 13] avg=32768.0000 max=65513 max_rank=1 min=13 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=25 tokens_per_ep_rank=[32455, 32796, 32753, 33096, 32488, 33096, 32686, 32774] avg=32768.0000 max=33096 max_rank=3 min=32455 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65081 max_load_reduction_pct=0.662895 moved_to_replicas=70840 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=29 tokens_per_ep_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] avg=32768.0000 max=33096 max_rank=1 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=97205 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=31 tokens_per_ep_rank=[32803, 65486, 32517, 65486, 2, 32794, 32440, 616] avg=32768.0000 max=65486 max_rank=1 min=2 min_rank=4 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=28 tokens_per_ep_rank=[32701, 65453, 54, 32761, 65480, 86, 89, 65520] avg=32768.0000 max=65520 max_rank=7 min=54 min_rank=2 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=30 tokens_per_ep_rank=[46, 65443, 32722, 32799, 32801, 65480, 32726, 127] avg=32768.0000 max=65480 max_rank=5 min=46 min_rank=0 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=64731 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=29 tokens_per_ep_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] avg=32768.0000 max=33096 max_rank=1 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=97205 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=29 tokens_per_ep_rank=[113, 65513, 65500, 32725, 65480, 15, 32785, 13] avg=32768.0000 max=65513 max_rank=1 min=13 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=31 tokens_per_ep_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] avg=32768.0000 max=33096 max_rank=1 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=64780 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=28 tokens_per_ep_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] avg=32768.0000 max=33096 max_rank=1 min=32446 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=97165 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=64731 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=30 tokens_per_ep_rank=[46, 65443, 32722, 32799, 32801, 65480, 32726, 127] avg=32768.0000 max=65480 max_rank=5 min=46 min_rank=0 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=26 tokens_per_ep_rank=[32794, 30, 33614, 46, 64825, 32701, 65392, 32742] avg=32768.0000 max=65392 max_rank=6 min=30 min_rank=1 imbalance_ratio=1.995605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=29 tokens_per_ep_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] avg=32768.0000 max=33096 max_rank=1 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=97205 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=31 tokens_per_ep_rank=[32803, 65486, 32517, 65486, 2, 32794, 32440, 616] avg=32768.0000 max=65486 max_rank=1 min=2 min_rank=4 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=30 tokens_per_ep_rank=[46, 65443, 32722, 32799, 32801, 65480, 32726, 127] avg=32768.0000 max=65480 max_rank=5 min=46 min_rank=0 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=64731 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=32 tokens_per_ep_rank=[17483, 571, 32756, 98225, 32788, 75, 7, 80239] avg=32768.0000 max=98225 max_rank=3 min=7 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=26 tokens_per_ep_rank=[32794, 32326, 33096, 32293, 33096, 32701, 33096, 32742] avg=32768.0000 max=33096 max_rank=2 min=32293 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32296 max_load_reduction_pct=0.493883 moved_to_replicas=64543 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=31 tokens_per_ep_rank=[32803, 65486, 32517, 65486, 2, 32794, 32440, 616] avg=32768.0000 max=65486 max_rank=1 min=2 min_rank=4 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=29 tokens_per_ep_rank=[113, 65513, 65500, 32725, 65480, 15, 32785, 13] avg=32768.0000 max=65513 max_rank=1 min=13 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=31 tokens_per_ep_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] avg=32768.0000 max=33096 max_rank=1 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=64780 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=64731 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=30 tokens_per_ep_rank=[46, 65443, 32722, 32799, 32801, 65480, 32726, 127] avg=32768.0000 max=65480 max_rank=5 min=46 min_rank=0 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=32 tokens_per_ep_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] avg=32768.0000 max=33096 max_rank=1 min=32102 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=112272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=31 tokens_per_ep_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] avg=32768.0000 max=33096 max_rank=1 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=64780 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=29 tokens_per_ep_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] avg=32768.0000 max=33096 max_rank=1 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=97205 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=31 tokens_per_ep_rank=[32803, 65486, 32517, 65486, 2, 32794, 32440, 616] avg=32768.0000 max=65486 max_rank=1 min=2 min_rank=4 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=64731 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 79, 32862, 32799, 65486, 32741] avg=32768.0000 max=65486 max_rank=6 min=79 min_rank=3 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=32 tokens_per_ep_rank=[17483, 571, 32756, 98225, 32788, 75, 7, 80239] avg=32768.0000 max=98225 max_rank=3 min=7 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=31 tokens_per_ep_rank=[32803, 65486, 32517, 65486, 2, 32794, 32440, 616] avg=32768.0000 max=65486 max_rank=1 min=2 min_rank=4 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=33 tokens_per_ep_rank=[65465, 65443, 42, 32783, 32736, 32803, 32824, 48] avg=32768.0000 max=65465 max_rank=0 min=42 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=31 tokens_per_ep_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] avg=32768.0000 max=33096 max_rank=1 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=64780 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=32 tokens_per_ep_rank=[17483, 571, 32756, 98225, 32788, 75, 7, 80239] avg=32768.0000 max=98225 max_rank=3 min=7 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=27 tokens_per_ep_rank=[32805, 32605, 32767, 32469, 32862, 32799, 33096, 32741] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=32390 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=30 tokens_per_ep_rank=[46, 65443, 32722, 32799, 32801, 65480, 32726, 127] avg=32768.0000 max=65480 max_rank=5 min=46 min_rank=0 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=32 tokens_per_ep_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] avg=32768.0000 max=33096 max_rank=1 min=32102 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=112272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=31 tokens_per_ep_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] avg=32768.0000 max=33096 max_rank=1 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=64780 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=31 tokens_per_ep_rank=[32803, 65486, 32517, 65486, 2, 32794, 32440, 616] avg=32768.0000 max=65486 max_rank=1 min=2 min_rank=4 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] avg=32768.0000 max=33096 max_rank=0 min=32395 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=64716 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=32 tokens_per_ep_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] avg=32768.0000 max=33096 max_rank=1 min=32102 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=112272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=32 tokens_per_ep_rank=[17483, 571, 32756, 98225, 32788, 75, 7, 80239] avg=32768.0000 max=98225 max_rank=3 min=7 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=64731 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=31 tokens_per_ep_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] avg=32768.0000 max=33096 max_rank=1 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=64780 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=28 tokens_per_ep_rank=[32701, 65453, 54, 32761, 65480, 86, 89, 65520] avg=32768.0000 max=65520 max_rank=7 min=54 min_rank=2 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=33 tokens_per_ep_rank=[65465, 65443, 42, 32783, 32736, 32803, 32824, 48] avg=32768.0000 max=65465 max_rank=0 min=42 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=32 tokens_per_ep_rank=[17483, 571, 32756, 98225, 32788, 75, 7, 80239] avg=32768.0000 max=98225 max_rank=3 min=7 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=34 tokens_per_ep_rank=[98011, 88, 105, 55, 32812, 5, 32992, 98076] avg=32768.0000 max=98076 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=32 tokens_per_ep_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] avg=32768.0000 max=33096 max_rank=1 min=32102 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=112272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=33 tokens_per_ep_rank=[65465, 65443, 42, 32783, 32736, 32803, 32824, 48] avg=32768.0000 max=65465 max_rank=0 min=42 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] avg=32768.0000 max=33096 max_rank=0 min=32395 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=64716 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=28 tokens_per_ep_rank=[32701, 33096, 32478, 32761, 33096, 32470, 32446, 33096] avg=32768.0000 max=33096 max_rank=1 min=32446 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=97165 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=31 tokens_per_ep_rank=[32803, 65486, 32517, 65486, 2, 32794, 32440, 616] avg=32768.0000 max=65486 max_rank=1 min=2 min_rank=4 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=32 tokens_per_ep_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] avg=32768.0000 max=33096 max_rank=1 min=32102 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=112272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=32 tokens_per_ep_rank=[17483, 571, 32756, 98225, 32788, 75, 7, 80239] avg=32768.0000 max=98225 max_rank=3 min=7 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] avg=32768.0000 max=33096 max_rank=0 min=32300 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64980 max_load_reduction_pct=0.662547 moved_to_replicas=129895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] avg=32768.0000 max=33096 max_rank=0 min=32395 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=64716 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=33 tokens_per_ep_rank=[65465, 65443, 42, 32783, 32736, 32803, 32824, 48] avg=32768.0000 max=65465 max_rank=0 min=42 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=31 tokens_per_ep_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] avg=32768.0000 max=33096 max_rank=1 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=64780 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=32 tokens_per_ep_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] avg=32768.0000 max=33096 max_rank=1 min=32102 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=112272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=34 tokens_per_ep_rank=[98011, 88, 105, 55, 32812, 5, 32992, 98076] avg=32768.0000 max=98076 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=29 tokens_per_ep_rank=[113, 65513, 65500, 32725, 65480, 15, 32785, 13] avg=32768.0000 max=65513 max_rank=1 min=13 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=33 tokens_per_ep_rank=[65465, 65443, 42, 32783, 32736, 32803, 32824, 48] avg=32768.0000 max=65465 max_rank=0 min=42 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=35 tokens_per_ep_rank=[32870, 56, 32809, 32794, 65394, 56, 32782, 65383] avg=32768.0000 max=65394 max_rank=4 min=56 min_rank=1 imbalance_ratio=1.995667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] avg=32768.0000 max=33096 max_rank=0 min=32395 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=64716 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=34 tokens_per_ep_rank=[98011, 88, 105, 55, 32812, 5, 32992, 98076] avg=32768.0000 max=98076 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] avg=32768.0000 max=33096 max_rank=0 min=32300 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64980 max_load_reduction_pct=0.662547 moved_to_replicas=129895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=29 tokens_per_ep_rank=[32497, 33096, 33096, 32725, 33096, 32419, 32785, 32430] avg=32768.0000 max=33096 max_rank=1 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=97205 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] avg=32768.0000 max=33096 max_rank=0 min=32395 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=64716 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=33 tokens_per_ep_rank=[65465, 65443, 42, 32783, 32736, 32803, 32824, 48] avg=32768.0000 max=65465 max_rank=0 min=42 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=32 tokens_per_ep_rank=[17483, 571, 32756, 98225, 32788, 75, 7, 80239] avg=32768.0000 max=98225 max_rank=3 min=7 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=35 tokens_per_ep_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32343 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32298 max_load_reduction_pct=0.493899 moved_to_replicas=64585 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] avg=32768.0000 max=33096 max_rank=0 min=32300 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64980 max_load_reduction_pct=0.662547 moved_to_replicas=129895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=34 tokens_per_ep_rank=[98011, 88, 105, 55, 32812, 5, 32992, 98076] avg=32768.0000 max=98076 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] avg=32768.0000 max=33096 max_rank=0 min=32395 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=64716 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=32 tokens_per_ep_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] avg=32768.0000 max=33096 max_rank=1 min=32102 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=112272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=35 tokens_per_ep_rank=[32870, 56, 32809, 32794, 65394, 56, 32782, 65383] avg=32768.0000 max=65394 max_rank=4 min=56 min_rank=1 imbalance_ratio=1.995667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=30 tokens_per_ep_rank=[46, 65443, 32722, 32799, 32801, 65480, 32726, 127] avg=32768.0000 max=65480 max_rank=5 min=46 min_rank=0 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=34 tokens_per_ep_rank=[98011, 88, 105, 55, 32812, 5, 32992, 98076] avg=32768.0000 max=98076 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=36 tokens_per_ep_rank=[65386, 65720, 26, 16, 97, 88, 98070, 32741] avg=32768.0000 max=98070 max_rank=6 min=16 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] avg=32768.0000 max=33096 max_rank=0 min=32300 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64980 max_load_reduction_pct=0.662547 moved_to_replicas=129895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=35 tokens_per_ep_rank=[32870, 56, 32809, 32794, 65394, 56, 32782, 65383] avg=32768.0000 max=65394 max_rank=4 min=56 min_rank=1 imbalance_ratio=1.995667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=35 tokens_per_ep_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32343 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32298 max_load_reduction_pct=0.493899 moved_to_replicas=64585 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] avg=32768.0000 max=33096 max_rank=0 min=32300 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64980 max_load_reduction_pct=0.662547 moved_to_replicas=129895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=30 tokens_per_ep_rank=[32430, 33096, 32722, 32799, 32801, 33096, 32726, 32474] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=64731 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=34 tokens_per_ep_rank=[98011, 88, 105, 55, 32812, 5, 32992, 98076] avg=32768.0000 max=98076 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] avg=32768.0000 max=33096 max_rank=0 min=32279 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=129888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=33 tokens_per_ep_rank=[65465, 65443, 42, 32783, 32736, 32803, 32824, 48] avg=32768.0000 max=65465 max_rank=0 min=42 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=35 tokens_per_ep_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32343 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32298 max_load_reduction_pct=0.493899 moved_to_replicas=64585 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=35 tokens_per_ep_rank=[32870, 56, 32809, 32794, 65394, 56, 32782, 65383] avg=32768.0000 max=65394 max_rank=4 min=56 min_rank=1 imbalance_ratio=1.995667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] avg=32768.0000 max=33096 max_rank=0 min=32300 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64980 max_load_reduction_pct=0.662547 moved_to_replicas=129895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=36 tokens_per_ep_rank=[65386, 65720, 26, 16, 97, 88, 98070, 32741] avg=32768.0000 max=98070 max_rank=6 min=16 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] avg=32768.0000 max=33096 max_rank=0 min=32395 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=64716 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=35 tokens_per_ep_rank=[32870, 56, 32809, 32794, 65394, 56, 32782, 65383] avg=32768.0000 max=65394 max_rank=4 min=56 min_rank=1 imbalance_ratio=1.995667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=31 tokens_per_ep_rank=[32803, 65486, 32517, 65486, 2, 32794, 32440, 616] avg=32768.0000 max=65486 max_rank=1 min=2 min_rank=4 imbalance_ratio=1.998474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=37 tokens_per_ep_rank=[65417, 101, 122, 65433, 849, 3, 9, 130210] avg=32768.0000 max=130210 max_rank=7 min=3 min_rank=5 imbalance_ratio=3.973694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=36 tokens_per_ep_rank=[65386, 65720, 26, 16, 97, 88, 98070, 32741] avg=32768.0000 max=98070 max_rank=6 min=16 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=35 tokens_per_ep_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32343 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32298 max_load_reduction_pct=0.493899 moved_to_replicas=64585 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] avg=32768.0000 max=33096 max_rank=0 min=32279 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=129888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] avg=32768.0000 max=33096 max_rank=0 min=31855 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97114 max_load_reduction_pct=0.745826 moved_to_replicas=161772 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=35 tokens_per_ep_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32343 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32298 max_load_reduction_pct=0.493899 moved_to_replicas=64585 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=31 tokens_per_ep_rank=[32803, 33096, 32517, 33096, 32392, 32794, 32440, 33006] avg=32768.0000 max=33096 max_rank=1 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32390 max_load_reduction_pct=0.494610 moved_to_replicas=64780 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=35 tokens_per_ep_rank=[32870, 56, 32809, 32794, 65394, 56, 32782, 65383] avg=32768.0000 max=65394 max_rank=4 min=56 min_rank=1 imbalance_ratio=1.995667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=34 tokens_per_ep_rank=[98011, 88, 105, 55, 32812, 5, 32992, 98076] avg=32768.0000 max=98076 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] avg=32768.0000 max=33096 max_rank=0 min=32279 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=129888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=36 tokens_per_ep_rank=[65386, 65720, 26, 16, 97, 88, 98070, 32741] avg=32768.0000 max=98070 max_rank=6 min=16 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=37 tokens_per_ep_rank=[65417, 101, 122, 65433, 849, 3, 9, 130210] avg=32768.0000 max=130210 max_rank=7 min=3 min_rank=5 imbalance_ratio=3.973694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=35 tokens_per_ep_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32343 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32298 max_load_reduction_pct=0.493899 moved_to_replicas=64585 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] avg=32768.0000 max=33096 max_rank=0 min=32300 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64980 max_load_reduction_pct=0.662547 moved_to_replicas=129895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=38 tokens_per_ep_rank=[32825, 65442, 152, 65417, 130, 0, 42, 98136] avg=32768.0000 max=98136 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=36 tokens_per_ep_rank=[65386, 65720, 26, 16, 97, 88, 98070, 32741] avg=32768.0000 max=98070 max_rank=6 min=16 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=32 tokens_per_ep_rank=[17483, 571, 32756, 98225, 32788, 75, 7, 80239] avg=32768.0000 max=98225 max_rank=3 min=7 min_rank=6 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] avg=32768.0000 max=33096 max_rank=0 min=32279 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=129888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] avg=32768.0000 max=33096 max_rank=0 min=31855 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97114 max_load_reduction_pct=0.745826 moved_to_replicas=161772 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=37 tokens_per_ep_rank=[65417, 101, 122, 65433, 849, 3, 9, 130210] avg=32768.0000 max=130210 max_rank=7 min=3 min_rank=5 imbalance_ratio=3.973694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=38 tokens_per_ep_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] avg=32768.0000 max=33096 max_rank=1 min=32363 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65040 max_load_reduction_pct=0.662754 moved_to_replicas=129707 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] avg=32768.0000 max=33096 max_rank=0 min=32279 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=129888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=32 tokens_per_ep_rank=[32102, 33096, 32756, 33096, 32788, 32490, 32721, 33095] avg=32768.0000 max=33096 max_rank=1 min=32102 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=112272 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=36 tokens_per_ep_rank=[65386, 65720, 26, 16, 97, 88, 98070, 32741] avg=32768.0000 max=98070 max_rank=6 min=16 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=35 tokens_per_ep_rank=[32870, 56, 32809, 32794, 65394, 56, 32782, 65383] avg=32768.0000 max=65394 max_rank=4 min=56 min_rank=1 imbalance_ratio=1.995667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] avg=32768.0000 max=33096 max_rank=0 min=31855 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97114 max_load_reduction_pct=0.745826 moved_to_replicas=161772 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=37 tokens_per_ep_rank=[65417, 101, 122, 65433, 849, 3, 9, 130210] avg=32768.0000 max=130210 max_rank=7 min=3 min_rank=5 imbalance_ratio=3.973694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=38 tokens_per_ep_rank=[32825, 65442, 152, 65417, 130, 0, 42, 98136] avg=32768.0000 max=98136 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 43, 495, 32764, 32863, 98150, 32737] avg=32768.0000 max=98150 max_rank=6 min=43 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] avg=32768.0000 max=33096 max_rank=0 min=32279 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=129888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=37 tokens_per_ep_rank=[65417, 101, 122, 65433, 849, 3, 9, 130210] avg=32768.0000 max=130210 max_rank=7 min=3 min_rank=5 imbalance_ratio=3.973694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=35 tokens_per_ep_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32343 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32298 max_load_reduction_pct=0.493899 moved_to_replicas=64585 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] avg=32768.0000 max=33096 max_rank=0 min=31855 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97114 max_load_reduction_pct=0.745826 moved_to_replicas=161772 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=38 tokens_per_ep_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] avg=32768.0000 max=33096 max_rank=1 min=32363 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65040 max_load_reduction_pct=0.662754 moved_to_replicas=129707 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=33 tokens_per_ep_rank=[65465, 65443, 42, 32783, 32736, 32803, 32824, 48] avg=32768.0000 max=65465 max_rank=0 min=42 min_rank=2 imbalance_ratio=1.997833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=38 tokens_per_ep_rank=[32825, 65442, 152, 65417, 130, 0, 42, 98136] avg=32768.0000 max=98136 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] avg=32768.0000 max=33096 max_rank=6 min=32368 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] avg=32768.0000 max=33096 max_rank=0 min=31855 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97114 max_load_reduction_pct=0.745826 moved_to_replicas=161772 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=37 tokens_per_ep_rank=[65417, 101, 122, 65433, 849, 3, 9, 130210] avg=32768.0000 max=130210 max_rank=7 min=3 min_rank=5 imbalance_ratio=3.973694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=36 tokens_per_ep_rank=[65386, 65720, 26, 16, 97, 88, 98070, 32741] avg=32768.0000 max=98070 max_rank=6 min=16 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=38 tokens_per_ep_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] avg=32768.0000 max=33096 max_rank=1 min=32363 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65040 max_load_reduction_pct=0.662754 moved_to_replicas=129707 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=33 tokens_per_ep_rank=[33096, 33096, 32411, 32783, 32736, 32803, 32824, 32395] avg=32768.0000 max=33096 max_rank=0 min=32395 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32369 max_load_reduction_pct=0.494447 moved_to_replicas=64716 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=38 tokens_per_ep_rank=[32825, 65442, 152, 65417, 130, 0, 42, 98136] avg=32768.0000 max=98136 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 43, 495, 32764, 32863, 98150, 32737] avg=32768.0000 max=98150 max_rank=6 min=43 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 30916, 32730, 67311, 29] avg=32768.0000 max=67311 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.054169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] avg=32768.0000 max=33096 max_rank=0 min=31855 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97114 max_load_reduction_pct=0.745826 moved_to_replicas=161772 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=38 tokens_per_ep_rank=[32825, 65442, 152, 65417, 130, 0, 42, 98136] avg=32768.0000 max=98136 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] avg=32768.0000 max=33096 max_rank=0 min=32279 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=129888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=38 tokens_per_ep_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] avg=32768.0000 max=33096 max_rank=1 min=32363 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65040 max_load_reduction_pct=0.662754 moved_to_replicas=129707 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] avg=32768.0000 max=33096 max_rank=6 min=32368 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 43, 495, 32764, 32863, 98150, 32737] avg=32768.0000 max=98150 max_rank=6 min=43 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34215 max_load_reduction_pct=0.508312 moved_to_replicas=34215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=34 tokens_per_ep_rank=[98011, 88, 105, 55, 32812, 5, 32992, 98076] avg=32768.0000 max=98076 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=38 tokens_per_ep_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] avg=32768.0000 max=33096 max_rank=1 min=32363 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65040 max_load_reduction_pct=0.662754 moved_to_replicas=129707 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=38 tokens_per_ep_rank=[32825, 65442, 152, 65417, 130, 0, 42, 98136] avg=32768.0000 max=98136 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] avg=32768.0000 max=33096 max_rank=6 min=32368 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=37 tokens_per_ep_rank=[65417, 101, 122, 65433, 849, 3, 9, 130210] avg=32768.0000 max=130210 max_rank=7 min=3 min_rank=5 imbalance_ratio=3.973694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 43, 495, 32764, 32863, 98150, 32737] avg=32768.0000 max=98150 max_rank=6 min=43 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=34 tokens_per_ep_rank=[33096, 32808, 32300, 32317, 32812, 32723, 32992, 33096] avg=32768.0000 max=33096 max_rank=0 min=32300 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64980 max_load_reduction_pct=0.662547 moved_to_replicas=129895 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 30916, 32730, 67311, 29] avg=32768.0000 max=67311 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.054169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=41 tokens_per_ep_rank=[68, 65489, 32808, 65216, 65489, 0, 138, 32936] avg=32768.0000 max=65489 max_rank=1 min=0 min_rank=5 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=38 tokens_per_ep_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] avg=32768.0000 max=33096 max_rank=1 min=32363 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65040 max_load_reduction_pct=0.662754 moved_to_replicas=129707 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 43, 495, 32764, 32863, 98150, 32737] avg=32768.0000 max=98150 max_rank=6 min=43 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] avg=32768.0000 max=33096 max_rank=0 min=31855 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97114 max_load_reduction_pct=0.745826 moved_to_replicas=161772 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] avg=32768.0000 max=33096 max_rank=6 min=32368 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34215 max_load_reduction_pct=0.508312 moved_to_replicas=34215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 30916, 32730, 67311, 29] avg=32768.0000 max=67311 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.054169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=41 tokens_per_ep_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] avg=32768.0000 max=33096 max_rank=1 min=32258 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=96906 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=35 tokens_per_ep_rank=[32870, 56, 32809, 32794, 65394, 56, 32782, 65383] avg=32768.0000 max=65394 max_rank=4 min=56 min_rank=1 imbalance_ratio=1.995667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] avg=32768.0000 max=33096 max_rank=6 min=32368 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 43, 495, 32764, 32863, 98150, 32737] avg=32768.0000 max=98150 max_rank=6 min=43 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34215 max_load_reduction_pct=0.508312 moved_to_replicas=34215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 30916, 32730, 67311, 29] avg=32768.0000 max=67311 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.054169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=38 tokens_per_ep_rank=[32825, 65442, 152, 65417, 130, 0, 42, 98136] avg=32768.0000 max=98136 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=41 tokens_per_ep_rank=[68, 65489, 32808, 65216, 65489, 0, 138, 32936] avg=32768.0000 max=65489 max_rank=1 min=0 min_rank=5 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=35 tokens_per_ep_rank=[32870, 32354, 32809, 32794, 33096, 32343, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32343 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32298 max_load_reduction_pct=0.493899 moved_to_replicas=64585 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=42 tokens_per_ep_rank=[65700, 0, 92, 32779, 65350, 65342, 32821, 60] avg=32768.0000 max=65700 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] avg=32768.0000 max=33096 max_rank=6 min=32368 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 30916, 32730, 67311, 29] avg=32768.0000 max=67311 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.054169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=41 tokens_per_ep_rank=[68, 65489, 32808, 65216, 65489, 0, 138, 32936] avg=32768.0000 max=65489 max_rank=1 min=0 min_rank=5 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34215 max_load_reduction_pct=0.508312 moved_to_replicas=34215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=38 tokens_per_ep_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] avg=32768.0000 max=33096 max_rank=1 min=32363 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65040 max_load_reduction_pct=0.662754 moved_to_replicas=129707 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=41 tokens_per_ep_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] avg=32768.0000 max=33096 max_rank=1 min=32258 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=96906 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] avg=32768.0000 max=33096 max_rank=0 min=32314 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32604 max_load_reduction_pct=0.496256 moved_to_replicas=97104 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34215 max_load_reduction_pct=0.508312 moved_to_replicas=34215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=36 tokens_per_ep_rank=[65386, 65720, 26, 16, 97, 88, 98070, 32741] avg=32768.0000 max=98070 max_rank=6 min=16 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=41 tokens_per_ep_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] avg=32768.0000 max=33096 max_rank=1 min=32258 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=96906 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 30916, 32730, 67311, 29] avg=32768.0000 max=67311 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.054169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=41 tokens_per_ep_rank=[68, 65489, 32808, 65216, 65489, 0, 138, 32936] avg=32768.0000 max=65489 max_rank=1 min=0 min_rank=5 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=42 tokens_per_ep_rank=[65700, 0, 92, 32779, 65350, 65342, 32821, 60] avg=32768.0000 max=65700 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 43, 495, 32764, 32863, 98150, 32737] avg=32768.0000 max=98150 max_rank=6 min=43 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=43 tokens_per_ep_rank=[54, 129971, 72, 105, 32805, 33, 32723, 66381] avg=32768.0000 max=129971 max_rank=1 min=33 min_rank=5 imbalance_ratio=3.966400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=36 tokens_per_ep_rank=[33096, 33096, 32279, 32737, 32387, 32712, 33096, 32741] avg=32768.0000 max=33096 max_rank=0 min=32279 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=129888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34215 max_load_reduction_pct=0.508312 moved_to_replicas=34215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=41 tokens_per_ep_rank=[68, 65489, 32808, 65216, 65489, 0, 138, 32936] avg=32768.0000 max=65489 max_rank=1 min=0 min_rank=5 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=42 tokens_per_ep_rank=[65700, 0, 92, 32779, 65350, 65342, 32821, 60] avg=32768.0000 max=65700 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=41 tokens_per_ep_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] avg=32768.0000 max=33096 max_rank=1 min=32258 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=96906 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] avg=32768.0000 max=33096 max_rank=0 min=32314 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32604 max_load_reduction_pct=0.496256 moved_to_replicas=97104 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] avg=32768.0000 max=33096 max_rank=6 min=32368 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] avg=32768.0000 max=33096 max_rank=1 min=32096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96875 max_load_reduction_pct=0.745359 moved_to_replicas=130160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=41 tokens_per_ep_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] avg=32768.0000 max=33096 max_rank=1 min=32258 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=96906 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=37 tokens_per_ep_rank=[65417, 101, 122, 65433, 849, 3, 9, 130210] avg=32768.0000 max=130210 max_rank=7 min=3 min_rank=5 imbalance_ratio=3.973694 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] avg=32768.0000 max=33096 max_rank=0 min=32314 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32604 max_load_reduction_pct=0.496256 moved_to_replicas=97104 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=41 tokens_per_ep_rank=[68, 65489, 32808, 65216, 65489, 0, 138, 32936] avg=32768.0000 max=65489 max_rank=1 min=0 min_rank=5 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=42 tokens_per_ep_rank=[65700, 0, 92, 32779, 65350, 65342, 32821, 60] avg=32768.0000 max=65700 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=43 tokens_per_ep_rank=[54, 129971, 72, 105, 32805, 33, 32723, 66381] avg=32768.0000 max=129971 max_rank=1 min=33 min_rank=5 imbalance_ratio=3.966400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=37 tokens_per_ep_rank=[33096, 31855, 32459, 33096, 33096, 32720, 32726, 33096] avg=32768.0000 max=33096 max_rank=0 min=31855 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97114 max_load_reduction_pct=0.745826 moved_to_replicas=161772 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 30916, 32730, 67311, 29] avg=32768.0000 max=67311 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.054169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 10, 32734, 47, 65471, 32877, 65474] avg=32768.0000 max=65474 max_rank=7 min=10 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=41 tokens_per_ep_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] avg=32768.0000 max=33096 max_rank=1 min=32258 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=96906 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=42 tokens_per_ep_rank=[65700, 0, 92, 32779, 65350, 65342, 32821, 60] avg=32768.0000 max=65700 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=43 tokens_per_ep_rank=[54, 129971, 72, 105, 32805, 33, 32723, 66381] avg=32768.0000 max=129971 max_rank=1 min=33 min_rank=5 imbalance_ratio=3.966400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] avg=32768.0000 max=33096 max_rank=0 min=32314 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32604 max_load_reduction_pct=0.496256 moved_to_replicas=97104 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] avg=32768.0000 max=33096 max_rank=1 min=32096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96875 max_load_reduction_pct=0.745359 moved_to_replicas=130160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=5 min=32388 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64753 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34215 max_load_reduction_pct=0.508312 moved_to_replicas=34215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] avg=32768.0000 max=33096 max_rank=0 min=32314 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32604 max_load_reduction_pct=0.496256 moved_to_replicas=97104 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] avg=32768.0000 max=33096 max_rank=1 min=32096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96875 max_load_reduction_pct=0.745359 moved_to_replicas=130160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=38 tokens_per_ep_rank=[32825, 65442, 152, 65417, 130, 0, 42, 98136] avg=32768.0000 max=98136 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=42 tokens_per_ep_rank=[65700, 0, 92, 32779, 65350, 65342, 32821, 60] avg=32768.0000 max=65700 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=43 tokens_per_ep_rank=[54, 129971, 72, 105, 32805, 33, 32723, 66381] avg=32768.0000 max=129971 max_rank=1 min=33 min_rank=5 imbalance_ratio=3.966400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 10, 32734, 47, 65471, 32877, 65474] avg=32768.0000 max=65474 max_rank=7 min=10 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=45 tokens_per_ep_rank=[32801, 30612, 105, 1, 100554, 32923, 32334, 32814] avg=32768.0000 max=100554 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.068665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=38 tokens_per_ep_rank=[32825, 33096, 32473, 33096, 32476, 32719, 32363, 33096] avg=32768.0000 max=33096 max_rank=1 min=32363 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65040 max_load_reduction_pct=0.662754 moved_to_replicas=129707 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=41 tokens_per_ep_rank=[68, 65489, 32808, 65216, 65489, 0, 138, 32936] avg=32768.0000 max=65489 max_rank=1 min=0 min_rank=5 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=43 tokens_per_ep_rank=[54, 129971, 72, 105, 32805, 33, 32723, 66381] avg=32768.0000 max=129971 max_rank=1 min=33 min_rank=5 imbalance_ratio=3.966400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] avg=32768.0000 max=33096 max_rank=0 min=32314 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32604 max_load_reduction_pct=0.496256 moved_to_replicas=97104 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 10, 32734, 47, 65471, 32877, 65474] avg=32768.0000 max=65474 max_rank=7 min=10 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] avg=32768.0000 max=33096 max_rank=1 min=32096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96875 max_load_reduction_pct=0.745359 moved_to_replicas=130160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=5 min=32388 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64753 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] avg=32768.0000 max=33096 max_rank=4 min=32334 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=67458 max_load_reduction_pct=0.670863 moved_to_replicas=67458 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=41 tokens_per_ep_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] avg=32768.0000 max=33096 max_rank=1 min=32258 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=96906 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] avg=32768.0000 max=33096 max_rank=1 min=32096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96875 max_load_reduction_pct=0.745359 moved_to_replicas=130160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=5 min=32388 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64753 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 43, 495, 32764, 32863, 98150, 32737] avg=32768.0000 max=98150 max_rank=6 min=43 min_rank=2 imbalance_ratio=2.995300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=43 tokens_per_ep_rank=[54, 129971, 72, 105, 32805, 33, 32723, 66381] avg=32768.0000 max=129971 max_rank=1 min=33 min_rank=5 imbalance_ratio=3.966400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 10, 32734, 47, 65471, 32877, 65474] avg=32768.0000 max=65474 max_rank=7 min=10 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=45 tokens_per_ep_rank=[32801, 30612, 105, 1, 100554, 32923, 32334, 32814] avg=32768.0000 max=100554 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.068665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=46 tokens_per_ep_rank=[19727, 93, 65494, 2, 23346, 87964, 51, 65467] avg=32768.0000 max=87964 max_rank=5 min=2 min_rank=3 imbalance_ratio=2.684448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=42 tokens_per_ep_rank=[65700, 0, 92, 32779, 65350, 65342, 32821, 60] avg=32768.0000 max=65700 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 10, 32734, 47, 65471, 32877, 65474] avg=32768.0000 max=65474 max_rank=7 min=10 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=45 tokens_per_ep_rank=[32801, 30612, 105, 1, 100554, 32923, 32334, 32814] avg=32768.0000 max=100554 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.068665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] avg=32768.0000 max=33096 max_rank=1 min=32096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96875 max_load_reduction_pct=0.745359 moved_to_replicas=130160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=39 tokens_per_ep_rank=[32368, 32724, 32761, 32831, 32764, 32863, 33096, 32737] avg=32768.0000 max=33096 max_rank=6 min=32368 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65054 max_load_reduction_pct=0.662802 moved_to_replicas=65054 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=5 min=32388 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64753 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] avg=32768.0000 max=33096 max_rank=4 min=32334 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=67458 max_load_reduction_pct=0.670863 moved_to_replicas=67458 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=46 tokens_per_ep_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31453 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54867 max_load_reduction_pct=0.623744 moved_to_replicas=119637 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=5 min=32388 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64753 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] avg=32768.0000 max=33096 max_rank=0 min=32314 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32604 max_load_reduction_pct=0.496256 moved_to_replicas=97104 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] avg=32768.0000 max=33096 max_rank=4 min=32334 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=67458 max_load_reduction_pct=0.670863 moved_to_replicas=67458 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 10, 32734, 47, 65471, 32877, 65474] avg=32768.0000 max=65474 max_rank=7 min=10 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=45 tokens_per_ep_rank=[32801, 30612, 105, 1, 100554, 32923, 32334, 32814] avg=32768.0000 max=100554 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.068665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 30916, 32730, 67311, 29] avg=32768.0000 max=67311 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.054169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=46 tokens_per_ep_rank=[19727, 93, 65494, 2, 23346, 87964, 51, 65467] avg=32768.0000 max=87964 max_rank=5 min=2 min_rank=3 imbalance_ratio=2.684448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=47 tokens_per_ep_rank=[89, 32855, 4, 32719, 32790, 65493, 98120, 74] avg=32768.0000 max=98120 max_rank=6 min=4 min_rank=2 imbalance_ratio=2.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=5 min=32388 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64753 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] avg=32768.0000 max=33096 max_rank=4 min=32334 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=67458 max_load_reduction_pct=0.670863 moved_to_replicas=67458 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=45 tokens_per_ep_rank=[32801, 30612, 105, 1, 100554, 32923, 32334, 32814] avg=32768.0000 max=100554 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.068665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=40 tokens_per_ep_rank=[32842, 32737, 32767, 32812, 32412, 32730, 33096, 32748] avg=32768.0000 max=33096 max_rank=6 min=32412 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34215 max_load_reduction_pct=0.508312 moved_to_replicas=34215 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=46 tokens_per_ep_rank=[19727, 93, 65494, 2, 23346, 87964, 51, 65467] avg=32768.0000 max=87964 max_rank=5 min=2 min_rank=3 imbalance_ratio=2.684448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=43 tokens_per_ep_rank=[54, 129971, 72, 105, 32805, 33, 32723, 66381] avg=32768.0000 max=129971 max_rank=1 min=33 min_rank=5 imbalance_ratio=3.966400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=46 tokens_per_ep_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31453 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54867 max_load_reduction_pct=0.623744 moved_to_replicas=119637 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=47 tokens_per_ep_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65024 max_load_reduction_pct=0.662699 moved_to_replicas=97421 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] avg=32768.0000 max=33096 max_rank=4 min=32334 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=67458 max_load_reduction_pct=0.670863 moved_to_replicas=67458 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=46 tokens_per_ep_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31453 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54867 max_load_reduction_pct=0.623744 moved_to_replicas=119637 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] avg=32768.0000 max=33096 max_rank=1 min=32096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96875 max_load_reduction_pct=0.745359 moved_to_replicas=130160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=45 tokens_per_ep_rank=[32801, 30612, 105, 1, 100554, 32923, 32334, 32814] avg=32768.0000 max=100554 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.068665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=46 tokens_per_ep_rank=[19727, 93, 65494, 2, 23346, 87964, 51, 65467] avg=32768.0000 max=87964 max_rank=5 min=2 min_rank=3 imbalance_ratio=2.684448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=41 tokens_per_ep_rank=[68, 65489, 32808, 65216, 65489, 0, 138, 32936] avg=32768.0000 max=65489 max_rank=1 min=0 min_rank=5 imbalance_ratio=1.998566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=48 tokens_per_ep_rank=[59, 32727, 32752, 32718, 61344, 32800, 36863, 32881] avg=32768.0000 max=61344 max_rank=4 min=59 min_rank=0 imbalance_ratio=1.872070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=47 tokens_per_ep_rank=[89, 32855, 4, 32719, 32790, 65493, 98120, 74] avg=32768.0000 max=98120 max_rank=6 min=4 min_rank=2 imbalance_ratio=2.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] avg=32768.0000 max=33096 max_rank=4 min=32334 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=67458 max_load_reduction_pct=0.670863 moved_to_replicas=67458 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=46 tokens_per_ep_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31453 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54867 max_load_reduction_pct=0.623744 moved_to_replicas=119637 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=46 tokens_per_ep_rank=[19727, 93, 65494, 2, 23346, 87964, 51, 65467] avg=32768.0000 max=87964 max_rank=5 min=2 min_rank=3 imbalance_ratio=2.684448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=41 tokens_per_ep_rank=[32461, 33096, 32808, 33096, 33096, 32393, 32258, 32936] avg=32768.0000 max=33096 max_rank=1 min=32258 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32393 max_load_reduction_pct=0.494633 moved_to_replicas=96906 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=47 tokens_per_ep_rank=[89, 32855, 4, 32719, 32790, 65493, 98120, 74] avg=32768.0000 max=98120 max_rank=6 min=4 min_rank=2 imbalance_ratio=2.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=48 tokens_per_ep_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] avg=32768.0000 max=33097 max_rank=4 min=32073 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28247 max_load_reduction_pct=0.460469 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=47 tokens_per_ep_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65024 max_load_reduction_pct=0.662699 moved_to_replicas=97421 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 10, 32734, 47, 65471, 32877, 65474] avg=32768.0000 max=65474 max_rank=7 min=10 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=46 tokens_per_ep_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31453 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54867 max_load_reduction_pct=0.623744 moved_to_replicas=119637 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=47 tokens_per_ep_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65024 max_load_reduction_pct=0.662699 moved_to_replicas=97421 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=46 tokens_per_ep_rank=[19727, 93, 65494, 2, 23346, 87964, 51, 65467] avg=32768.0000 max=87964 max_rank=5 min=2 min_rank=3 imbalance_ratio=2.684448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=5 min=32388 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64753 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=47 tokens_per_ep_rank=[89, 32855, 4, 32719, 32790, 65493, 98120, 74] avg=32768.0000 max=98120 max_rank=6 min=4 min_rank=2 imbalance_ratio=2.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=42 tokens_per_ep_rank=[65700, 0, 92, 32779, 65350, 65342, 32821, 60] avg=32768.0000 max=65700 max_rank=0 min=0 min_rank=1 imbalance_ratio=2.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=48 tokens_per_ep_rank=[59, 32727, 32752, 32718, 61344, 32800, 36863, 32881] avg=32768.0000 max=61344 max_rank=4 min=59 min_rank=0 imbalance_ratio=1.872070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=46 tokens_per_ep_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31453 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54867 max_load_reduction_pct=0.623744 moved_to_replicas=119637 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=47 tokens_per_ep_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65024 max_load_reduction_pct=0.662699 moved_to_replicas=97421 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=47 tokens_per_ep_rank=[89, 32855, 4, 32719, 32790, 65493, 98120, 74] avg=32768.0000 max=98120 max_rank=6 min=4 min_rank=2 imbalance_ratio=2.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=48 tokens_per_ep_rank=[59, 32727, 32752, 32718, 61344, 32800, 36863, 32881] avg=32768.0000 max=61344 max_rank=4 min=59 min_rank=0 imbalance_ratio=1.872070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=42 tokens_per_ep_rank=[33096, 32604, 32338, 32779, 33096, 33096, 32821, 32314] avg=32768.0000 max=33096 max_rank=0 min=32314 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32604 max_load_reduction_pct=0.496256 moved_to_replicas=97104 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=48 tokens_per_ep_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] avg=32768.0000 max=33097 max_rank=4 min=32073 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28247 max_load_reduction_pct=0.460469 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=45 tokens_per_ep_rank=[32801, 30612, 105, 1, 100554, 32923, 32334, 32814] avg=32768.0000 max=100554 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.068665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=47 tokens_per_ep_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65024 max_load_reduction_pct=0.662699 moved_to_replicas=97421 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=48 tokens_per_ep_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] avg=32768.0000 max=33097 max_rank=4 min=32073 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28247 max_load_reduction_pct=0.460469 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=47 tokens_per_ep_rank=[89, 32855, 4, 32719, 32790, 65493, 98120, 74] avg=32768.0000 max=98120 max_rank=6 min=4 min_rank=2 imbalance_ratio=2.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] avg=32768.0000 max=33096 max_rank=4 min=32334 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=67458 max_load_reduction_pct=0.670863 moved_to_replicas=67458 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=48 tokens_per_ep_rank=[59, 32727, 32752, 32718, 61344, 32800, 36863, 32881] avg=32768.0000 max=61344 max_rank=4 min=59 min_rank=0 imbalance_ratio=1.872070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=43 tokens_per_ep_rank=[54, 129971, 72, 105, 32805, 33, 32723, 66381] avg=32768.0000 max=129971 max_rank=1 min=33 min_rank=5 imbalance_ratio=3.966400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=47 tokens_per_ep_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65024 max_load_reduction_pct=0.662699 moved_to_replicas=97421 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=48 tokens_per_ep_rank=[59, 32727, 32752, 32718, 61344, 32800, 36863, 32881] avg=32768.0000 max=61344 max_rank=4 min=59 min_rank=0 imbalance_ratio=1.872070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=48 tokens_per_ep_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] avg=32768.0000 max=33097 max_rank=4 min=32073 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28247 max_load_reduction_pct=0.460469 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=43 tokens_per_ep_rank=[32762, 33096, 32096, 32815, 32805, 32751, 32723, 33096] avg=32768.0000 max=33096 max_rank=1 min=32096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96875 max_load_reduction_pct=0.745359 moved_to_replicas=130160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=46 tokens_per_ep_rank=[19727, 93, 65494, 2, 23346, 87964, 51, 65467] avg=32768.0000 max=87964 max_rank=5 min=2 min_rank=3 imbalance_ratio=2.684448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=48 tokens_per_ep_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] avg=32768.0000 max=33097 max_rank=4 min=32073 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28247 max_load_reduction_pct=0.460469 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=48 tokens_per_ep_rank=[59, 32727, 32752, 32718, 61344, 32800, 36863, 32881] avg=32768.0000 max=61344 max_rank=4 min=59 min_rank=0 imbalance_ratio=1.872070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=46 tokens_per_ep_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31453 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54867 max_load_reduction_pct=0.623744 moved_to_replicas=119637 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 10, 32734, 47, 65471, 32877, 65474] avg=32768.0000 max=65474 max_rank=7 min=10 min_rank=2 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=48 tokens_per_ep_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] avg=32768.0000 max=33097 max_rank=4 min=32073 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28247 max_load_reduction_pct=0.460469 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=44 tokens_per_ep_rank=[32745, 32786, 32388, 32734, 32422, 33096, 32877, 33096] avg=32768.0000 max=33096 max_rank=5 min=32388 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=64753 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=47 tokens_per_ep_rank=[89, 32855, 4, 32719, 32790, 65493, 98120, 74] avg=32768.0000 max=98120 max_rank=6 min=4 min_rank=2 imbalance_ratio=2.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=47 tokens_per_ep_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65024 max_load_reduction_pct=0.662699 moved_to_replicas=97421 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=45 tokens_per_ep_rank=[32801, 30612, 105, 1, 100554, 32923, 32334, 32814] avg=32768.0000 max=100554 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.068665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=45 tokens_per_ep_rank=[32801, 32642, 32815, 32719, 33096, 32923, 32334, 32814] avg=32768.0000 max=33096 max_rank=4 min=32334 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=67458 max_load_reduction_pct=0.670863 moved_to_replicas=67458 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=48 tokens_per_ep_rank=[59, 32727, 32752, 32718, 61344, 32800, 36863, 32881] avg=32768.0000 max=61344 max_rank=4 min=59 min_rank=0 imbalance_ratio=1.872070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=48 tokens_per_ep_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] avg=32768.0000 max=33097 max_rank=4 min=32073 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28247 max_load_reduction_pct=0.460469 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=46 tokens_per_ep_rank=[19727, 93, 65494, 2, 23346, 87964, 51, 65467] avg=32768.0000 max=87964 max_rank=5 min=2 min_rank=3 imbalance_ratio=2.684448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=46 tokens_per_ep_rank=[33097, 32491, 33096, 32720, 31453, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31453 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54867 max_load_reduction_pct=0.623744 moved_to_replicas=119637 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=47 tokens_per_ep_rank=[89, 32855, 4, 32719, 32790, 65493, 98120, 74] avg=32768.0000 max=98120 max_rank=6 min=4 min_rank=2 imbalance_ratio=2.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=47 tokens_per_ep_rank=[32486, 32855, 32721, 32719, 32790, 33096, 33096, 32381] avg=32768.0000 max=33096 max_rank=5 min=32381 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65024 max_load_reduction_pct=0.662699 moved_to_replicas=97421 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=48 tokens_per_ep_rank=[59, 32727, 32752, 32718, 61344, 32800, 36863, 32881] avg=32768.0000 max=61344 max_rank=4 min=59 min_rank=0 imbalance_ratio=1.872070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=48 tokens_per_ep_rank=[32073, 32727, 32752, 32718, 33097, 32800, 33096, 32881] avg=32768.0000 max=33097 max_rank=4 min=32073 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28247 max_load_reduction_pct=0.460469 moved_to_replicas=32015 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=48 total_expected_recv=1571392 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=367488 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=48 total_expected_recv=1572591 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366976 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=48 total_expected_recv=1576288 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=362240 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=48 total_expected_recv=1570674 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368768 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=48 total_expected_recv=1574490 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364544 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=48 total_expected_recv=1574154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=362752 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=48 total_expected_recv=1569100 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=371328 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=48 total_expected_recv=1574223 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366976 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:47:01] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 730.7 | throughput per GPU (TFLOP/s/GPU): 122.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.308485E+00 | loss scale: 1.0 | grad norm: 4.097 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31867 expected_overflow=0 exact_recv_per_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31701 expected_overflow=0 exact_recv_per_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31867 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31701 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31220 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=31727 expected_overflow=0 exact_recv_per_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=31727 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=31345 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=31345 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=31278 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32290 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=31278 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32290 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=31687 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=31687 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32195 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32195 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=30789 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=30789 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=30893 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=30893 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32268 expected_overflow=0 exact_recv_per_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32268 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32374 expected_overflow=0 exact_recv_per_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32374 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32182 expected_overflow=0 exact_recv_per_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32182 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=30864 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=30864 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=31528 expected_overflow=0 exact_recv_per_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=31528 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=31590 expected_overflow=0 exact_recv_per_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=31590 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32302 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32302 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=31556 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=31556 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=31618 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=31618 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32207 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32207 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=31688 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=40960 expected_recv=31627 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=31688 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33015 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33015 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=31385 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=31385 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32059 expected_overflow=0 exact_recv_per_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32059 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32108 expected_overflow=0 exact_recv_per_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32108 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=40960 expected_recv=31627 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32320 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=40960 expected_recv=31282 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=40960 expected_recv=31282 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32180 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=40960 expected_recv=31687 expected_overflow=0 exact_recv_per_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=40960 expected_recv=31687 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33074 expected_overflow=0 exact_recv_per_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33074 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=40960 expected_recv=31976 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32127 expected_overflow=0 exact_recv_per_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=40960 expected_recv=31976 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32127 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32398 expected_overflow=0 exact_recv_per_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32398 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32180 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32366 expected_overflow=0 exact_recv_per_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32241 expected_overflow=0 exact_recv_per_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32366 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32241 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32206 expected_overflow=0 exact_recv_per_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32303 expected_overflow=0 exact_recv_per_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32206 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32303 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32305 expected_overflow=0 exact_recv_per_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32305 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=40960 expected_recv=31849 expected_overflow=0 exact_recv_per_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32136 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32136 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32316 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32316 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33075 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33075 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32127 expected_overflow=0 exact_recv_per_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32127 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32390 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=40960 expected_recv=31849 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=40960 expected_recv=33057 expected_overflow=0 exact_recv_per_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=40960 expected_recv=33057 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32065 expected_overflow=0 exact_recv_per_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32065 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32479 expected_overflow=0 exact_recv_per_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32479 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32137 expected_overflow=0 exact_recv_per_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32137 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=40960 expected_recv=31378 expected_overflow=0 exact_recv_per_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=40960 expected_recv=31378 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32390 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=40960 expected_recv=31604 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32133 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=40960 expected_recv=31604 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32133 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32269 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32269 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32454 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32454 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32479 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32479 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=40960 expected_recv=31987 expected_overflow=0 exact_recv_per_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=40960 expected_recv=31987 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32550 expected_overflow=0 exact_recv_per_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32550 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=40960 expected_recv=31672 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=40960 expected_recv=31672 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32629 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32629 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33061 expected_overflow=0 exact_recv_per_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32503 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32503 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32418 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32418 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32468 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32468 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=27 calls=1 budget=40960 expected_recv=31782 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=27 calls=1 budget=40960 expected_recv=31782 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33061 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33026 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33026 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32320 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32289 expected_overflow=0 exact_recv_per_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32289 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32143 expected_overflow=0 exact_recv_per_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32143 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32413 expected_overflow=0 exact_recv_per_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32413 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32387 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32285 expected_overflow=0 exact_recv_per_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32387 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32285 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32189 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32189 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32484 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32484 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32938 expected_overflow=0 exact_recv_per_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32938 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33067 expected_overflow=0 exact_recv_per_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32402 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32402 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33067 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32206 expected_overflow=0 exact_recv_per_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32263 expected_overflow=0 exact_recv_per_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32206 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32263 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32353 expected_overflow=0 exact_recv_per_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32353 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32403 expected_overflow=0 exact_recv_per_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32403 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32336 expected_overflow=0 exact_recv_per_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=31 calls=1 budget=40960 expected_recv=31919 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=31 calls=1 budget=40960 expected_recv=31919 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32336 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32327 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=28 calls=1 budget=40960 expected_recv=31441 expected_overflow=0 exact_recv_per_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32327 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=28 calls=1 budget=40960 expected_recv=31441 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32382 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32415 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32382 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32415 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32183 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32183 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32252 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32252 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=33 calls=1 budget=40960 expected_recv=31649 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=33 calls=1 budget=40960 expected_recv=31649 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32308 expected_overflow=0 exact_recv_per_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32308 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32510 expected_overflow=0 exact_recv_per_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33038 expected_overflow=0 exact_recv_per_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32510 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=34 calls=1 budget=40960 expected_recv=33038 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32353 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32353 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32488 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32488 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32162 expected_overflow=0 exact_recv_per_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32409 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32409 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32030 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32030 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32162 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32099 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32099 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=39 calls=1 budget=40960 expected_recv=31801 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=39 calls=1 budget=40960 expected_recv=31801 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32086 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32394 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32394 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32086 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32340 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32340 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32152 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32152 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32452 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32452 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32407 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32407 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=43 calls=1 budget=40960 expected_recv=31968 expected_overflow=0 exact_recv_per_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=43 calls=1 budget=40960 expected_recv=31968 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32262 expected_overflow=0 exact_recv_per_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32262 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32416 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32415 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32416 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32415 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32413 expected_overflow=0 exact_recv_per_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32915 expected_overflow=0 exact_recv_per_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32915 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32400 expected_overflow=0 exact_recv_per_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32413 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32400 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32348, 17809, 40144, 24951, 41793, 26854, 42394, 35851] avg=32768.0000 max=42394 max_rank=6 min=17809 min_rank=1 imbalance_ratio=1.293762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32348, 33097, 33095, 33097, 33095, 31220, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31220 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9297 max_load_reduction_pct=0.219300 moved_to_replicas=27798 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[28849, 15288, 49819, 27895, 2157, 20978, 52177, 64981] avg=32768.0000 max=64981 max_rank=7 min=2157 min_rank=4 imbalance_ratio=1.983063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[30472, 33096, 33096, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31884 max_load_reduction_pct=0.490667 moved_to_replicas=67689 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[34595, 442, 66054, 65045, 33159, 32763, 29556, 530] avg=32768.0000 max=66054 max_rank=2 min=442 min_rank=1 imbalance_ratio=2.015808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[33096, 32556, 33096, 33096, 33096, 32763, 31345, 33096] avg=32768.0000 max=33096 max_rank=0 min=31345 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32958 max_load_reduction_pct=0.498955 moved_to_replicas=66469 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[14682, 191, 32290, 58404, 2159, 57338, 63819, 33261] avg=32768.0000 max=63819 max_rank=6 min=191 min_rank=1 imbalance_ratio=1.947601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33096, 31278, 32290, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31278 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30723 max_load_reduction_pct=0.481408 moved_to_replicas=80438 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[44820, 47800, 3135, 32195, 2717, 865, 130570, 42] avg=32768.0000 max=130570 max_rank=6 min=42 min_rank=7 imbalance_ratio=3.984680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[33097, 33095, 31687, 32195, 33096, 33096, 33096, 32782] avg=32768.0000 max=33097 max_rank=0 min=31687 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=97473 max_load_reduction_pct=0.746519 moved_to_replicas=123902 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[9160, 1905, 23143, 8212, 33231, 58, 59036, 127399] avg=32768.0000 max=127399 max_rank=7 min=58 min_rank=5 imbalance_ratio=3.887909 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[33096, 33096, 30789, 33096, 33096, 32779, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94303 max_load_reduction_pct=0.740218 moved_to_replicas=120378 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[92082, 56, 30012, 32445, 32741, 42177, 54, 32577] avg=32768.0000 max=92082 max_rank=0 min=54 min_rank=6 imbalance_ratio=2.810120 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[33097, 33093, 32302, 32445, 32741, 33098, 32791, 32577] avg=32768.0000 max=33098 max_rank=5 min=32302 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=58984 max_load_reduction_pct=0.640560 moved_to_replicas=68067 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65914, 64741, 27, 32768, 30667, 32891, 32874, 2262] avg=32768.0000 max=65914 max_rank=0 min=27 min_rank=2 imbalance_ratio=2.011536 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32767, 32768, 31556, 32891, 32874, 33096] avg=32768.0000 max=33096 max_rank=0 min=31556 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32818 max_load_reduction_pct=0.497891 moved_to_replicas=64463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[468, 32742, 65422, 30, 88361, 50, 42139, 32932] avg=32768.0000 max=88361 max_rank=4 min=30 min_rank=3 imbalance_ratio=2.696564 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32794, 32742, 33096, 32770, 33097, 31618, 33095, 32932] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=55264 max_load_reduction_pct=0.625434 moved_to_replicas=96634 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[53360, 32769, 18, 22295, 65285, 34847, 53348, 222] avg=32768.0000 max=65285 max_rank=4 min=18 min_rank=2 imbalance_ratio=1.992340 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[33094, 32769, 32207, 31688, 33096, 33096, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31688 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32187 max_load_reduction_pct=0.493023 moved_to_replicas=74456 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[31627, 3184, 25, 129566, 91, 34323, 30481, 32847] avg=32768.0000 max=129566 max_rank=3 min=25 min_rank=2 imbalance_ratio=3.954041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[31627, 33096, 32765, 33096, 32830, 33096, 32787, 32847] avg=32768.0000 max=33096 max_rank=1 min=31627 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96470 max_load_reduction_pct=0.744563 moved_to_replicas=97697 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[44606, 65519, 32640, 53853, 54, 49, 32628, 32795] avg=32768.0000 max=65519 max_rank=1 min=49 min_rank=5 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[33097, 33096, 32640, 33096, 32320, 32472, 32628, 32795] avg=32768.0000 max=33097 max_rank=0 min=32320 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32422 max_load_reduction_pct=0.494849 moved_to_replicas=64690 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[1857, 65517, 32746, 65381, 46171, 481, 17161, 32830] avg=32768.0000 max=65517 max_rank=1 min=481 min_rank=5 imbalance_ratio=1.999420 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[33096, 33096, 32746, 33096, 33096, 32902, 31282, 32830] avg=32768.0000 max=33096 max_rank=0 min=31282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32421 max_load_reduction_pct=0.494849 moved_to_replicas=77781 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32727, 24601, 35962, 37809, 27, 65502, 65507, 9] avg=32768.0000 max=65507 max_rank=6 min=9 min_rank=7 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32727, 32180, 33096, 33096, 32433, 33096, 33096, 32420] avg=32768.0000 max=33096 max_rank=2 min=32180 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=72396 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[2445, 97902, 262, 32798, 1, 63363, 65303, 70] avg=32768.0000 max=97902 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.987732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32712, 33096, 32469, 32798, 32741, 33096, 33096, 32136] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64806 max_load_reduction_pct=0.661948 moved_to_replicas=127280 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32799, 0, 32744, 32757, 93646, 32909, 4546, 32743] avg=32768.0000 max=93646 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.857849 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32799, 32740, 32744, 32757, 33094, 32909, 32358, 32743] avg=32768.0000 max=33094 max_rank=4 min=32358 min_rank=6 imbalance_ratio=1.009949 max_load_reduction=60552 max_load_reduction_pct=0.646605 moved_to_replicas=60550 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[65420, 32316, 553, 32804, 65508, 32741, 32768, 34] avg=32768.0000 max=65508 max_rank=4 min=34 min_rank=7 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[33096, 32316, 32877, 32804, 33096, 32741, 32768, 32446] avg=32768.0000 max=33096 max_rank=0 min=32316 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64736 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[32742, 27, 32795, 32446, 32127, 65498, 29, 66480] avg=32768.0000 max=66480 max_rank=7 min=27 min_rank=1 imbalance_ratio=2.028809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[32742, 32767, 32795, 32446, 32127, 33096, 33075, 33096] avg=32768.0000 max=33096 max_rank=5 min=32127 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33384 max_load_reduction_pct=0.502166 moved_to_replicas=65786 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[65443, 65539, 511, 55, 27, 64975, 103, 65491] avg=32768.0000 max=65539 max_rank=1 min=27 min_rank=4 imbalance_ratio=2.000092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[33096, 33096, 32390, 32450, 32470, 33096, 32450, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[152, 65426, 32745, 32741, 54, 32768, 32771, 65487] avg=32768.0000 max=65487 max_rank=7 min=54 min_rank=4 imbalance_ratio=1.998505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[32482, 33096, 32745, 32741, 32445, 32768, 32771, 33096] avg=32768.0000 max=33096 max_rank=1 min=32445 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32391 max_load_reduction_pct=0.494617 moved_to_replicas=64721 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 65494, 96017, 54, 887, 2534, 32269] avg=32768.0000 max=96017 max_rank=3 min=54 min_rank=4 imbalance_ratio=2.930206 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[32133, 32756, 33096, 33096, 32792, 32906, 33096, 32269] avg=32768.0000 max=33096 max_rank=2 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62921 max_load_reduction_pct=0.655311 moved_to_replicas=95319 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[65515, 32977, 32528, 32804, 28, 65482, 32742, 68] avg=32768.0000 max=65515 max_rank=0 min=28 min_rank=4 imbalance_ratio=1.999359 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[33096, 32977, 32528, 32804, 32447, 33096, 32742, 32454] avg=32768.0000 max=33096 max_rank=0 min=32447 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64805 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[0, 32769, 65575, 32796, 32738, 19, 32692, 65555] avg=32768.0000 max=65575 max_rank=2 min=0 min_rank=0 imbalance_ratio=2.001190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[32479, 32769, 33096, 32796, 32738, 32478, 32692, 33096] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=64938 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[32768, 65481, 65504, 32739, 32766, 32741, 118, 27] avg=32768.0000 max=65504 max_rank=2 min=27 min_rank=7 imbalance_ratio=1.999023 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[32768, 33096, 33096, 32739, 32766, 32741, 32503, 32435] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32408 max_load_reduction_pct=0.494748 moved_to_replicas=64793 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=25 tokens_per_ep_rank=[32789, 56, 98223, 32741, 0, 65509, 31, 32795] avg=32768.0000 max=98223 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=25 tokens_per_ep_rank=[32789, 32469, 33096, 32741, 32740, 33096, 32418, 32795] avg=32768.0000 max=33096 max_rank=2 min=32418 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=97540 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=26 tokens_per_ep_rank=[320, 27, 98251, 4, 98215, 32566, 32738, 23] avg=32768.0000 max=98251 max_rank=2 min=4 min_rank=3 imbalance_ratio=2.998383 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=26 tokens_per_ep_rank=[32699, 32767, 33096, 32744, 33096, 32566, 32738, 32438] avg=32768.0000 max=33096 max_rank=2 min=32438 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65155 max_load_reduction_pct=0.663148 moved_to_replicas=130274 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33644, 238, 28, 65536, 31944, 65478, 32481] avg=32768.0000 max=65536 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=27 tokens_per_ep_rank=[32795, 33096, 32620, 32468, 33096, 32492, 33096, 32481] avg=32768.0000 max=33096 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=65370 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=28 tokens_per_ep_rank=[28, 98232, 54, 67, 21183, 47289, 32760, 62531] avg=32768.0000 max=98232 max_rank=1 min=28 min_rank=0 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=28 tokens_per_ep_rank=[32768, 33096, 32450, 31782, 33094, 33098, 32760, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=108764 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=29 tokens_per_ep_rank=[54, 29, 65533, 32770, 34547, 96543, 27, 32641] avg=32768.0000 max=96543 max_rank=5 min=27 min_rank=6 imbalance_ratio=2.946259 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=29 tokens_per_ep_rank=[32491, 32189, 33096, 32770, 33096, 33096, 32765, 32641] avg=32768.0000 max=33096 max_rank=2 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63447 max_load_reduction_pct=0.657189 moved_to_replicas=97335 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=30 tokens_per_ep_rank=[32765, 98205, 32769, 63760, 32768, 33, 24, 1820] avg=32768.0000 max=98205 max_rank=1 min=24 min_rank=6 imbalance_ratio=2.996979 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=30 tokens_per_ep_rank=[32765, 33096, 32769, 33096, 32768, 32402, 32764, 32484] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=95773 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 0, 65536, 32750, 32742, 32767, 32769] avg=32768.0000 max=65536 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=31 tokens_per_ep_rank=[32857, 32723, 32440, 33096, 32750, 32742, 32767, 32769] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=32 tokens_per_ep_rank=[65535, 32768, 39875, 65508, 32768, 25142, 5, 543] avg=32768.0000 max=65535 max_rank=0 min=5 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32768, 33098, 33096, 32768, 31919, 32444, 32955] avg=32768.0000 max=33098 max_rank=2 min=31919 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=32437 max_load_reduction_pct=0.494957 moved_to_replicas=71630 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=33 tokens_per_ep_rank=[65478, 65502, 0, 65541, 26, 65489, 81, 27] avg=32768.0000 max=65541 max_rank=3 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32445, 33096, 32432, 33096, 32463, 32420] avg=32768.0000 max=33096 max_rank=0 min=32420 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129626 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=34 tokens_per_ep_rank=[65482, 32649, 25, 29, 98193, 0, 32741, 33025] avg=32768.0000 max=98193 max_rank=4 min=0 min_rank=5 imbalance_ratio=2.996613 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=34 tokens_per_ep_rank=[33096, 32649, 32382, 32415, 33096, 32740, 32741, 33025] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=97483 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=35 tokens_per_ep_rank=[32795, 27, 35903, 57879, 3, 32183, 62819, 40535] avg=32768.0000 max=62819 max_rank=6 min=3 min_rank=4 imbalance_ratio=1.917084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=35 tokens_per_ep_rank=[32795, 32252, 33096, 33094, 32533, 32183, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=32183 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29723 max_load_reduction_pct=0.473153 moved_to_replicas=64752 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=36 tokens_per_ep_rank=[27, 131028, 29, 32766, 29, 32753, 32782, 32730] avg=32768.0000 max=131028 max_rank=1 min=27 min_rank=0 imbalance_ratio=3.998657 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=36 tokens_per_ep_rank=[32767, 33096, 32769, 32766, 32481, 32753, 32782, 32730] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97932 max_load_reduction_pct=0.747413 moved_to_replicas=97932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=37 tokens_per_ep_rank=[32741, 54, 55, 98222, 249, 32740, 59, 98024] avg=32768.0000 max=98222 max_rank=3 min=54 min_rank=1 imbalance_ratio=2.997498 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=37 tokens_per_ep_rank=[32741, 32794, 32441, 33096, 32437, 32740, 32799, 33096] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65126 max_load_reduction_pct=0.663049 moved_to_replicas=130054 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=38 tokens_per_ep_rank=[32767, 65482, 32748, 65502, 83, 0, 57, 65505] avg=32768.0000 max=65505 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=38 tokens_per_ep_rank=[32767, 33096, 32748, 33096, 32469, 32409, 32463, 33096] avg=32768.0000 max=33096 max_rank=1 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32409 max_load_reduction_pct=0.494756 moved_to_replicas=97201 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=39 tokens_per_ep_rank=[65455, 66221, 26, 32783, 34, 32833, 32762, 32030] avg=32768.0000 max=66221 max_rank=1 min=26 min_rank=2 imbalance_ratio=2.020905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=39 tokens_per_ep_rank=[33096, 33096, 32766, 32783, 32778, 32833, 32762, 32030] avg=32768.0000 max=33096 max_rank=0 min=32030 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33125 max_load_reduction_pct=0.500219 moved_to_replicas=65484 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=40 tokens_per_ep_rank=[433, 19796, 130896, 48600, 29, 29597, 32766, 27] avg=32768.0000 max=130896 max_rank=2 min=27 min_rank=7 imbalance_ratio=3.994629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=40 tokens_per_ep_rank=[32754, 33095, 33096, 33097, 32768, 31801, 32766, 32767] avg=32768.0000 max=33097 max_rank=3 min=31801 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=97799 max_load_reduction_pct=0.747150 moved_to_replicas=113304 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=41 tokens_per_ep_rank=[2770, 62156, 33365, 65481, 65507, 34, 82, 32749] avg=32768.0000 max=65507 max_rank=4 min=34 min_rank=5 imbalance_ratio=1.999115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=41 tokens_per_ep_rank=[32099, 33096, 33096, 33096, 33096, 32445, 32467, 32749] avg=32768.0000 max=33096 max_rank=1 min=32099 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32411 max_load_reduction_pct=0.494772 moved_to_replicas=94125 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=42 tokens_per_ep_rank=[65483, 32763, 32747, 28, 33103, 11, 97953, 56] avg=32768.0000 max=97953 max_rank=6 min=11 min_rank=5 imbalance_ratio=2.989288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=42 tokens_per_ep_rank=[33096, 32763, 32747, 32152, 33096, 32751, 33096, 32443] avg=32768.0000 max=33096 max_rank=0 min=32152 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.662124 moved_to_replicas=97251 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=43 tokens_per_ep_rank=[34, 130936, 27, 54, 32795, 32730, 3, 65565] avg=32768.0000 max=130936 max_rank=1 min=3 min_rank=6 imbalance_ratio=3.995850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=43 tokens_per_ep_rank=[32394, 33096, 32767, 32523, 32795, 32730, 32743, 33096] avg=32768.0000 max=33096 max_rank=1 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97840 max_load_reduction_pct=0.747235 moved_to_replicas=130309 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=44 tokens_per_ep_rank=[10, 56, 0, 32762, 32766, 65492, 65517, 65541] avg=32768.0000 max=65541 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.000153 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=44 tokens_per_ep_rank=[32431, 32452, 32445, 32762, 32766, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=97262 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=45 tokens_per_ep_rank=[32794, 22, 11, 32742, 65506, 98221, 53, 32795] avg=32768.0000 max=98221 max_rank=5 min=11 min_rank=2 imbalance_ratio=2.997467 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=45 tokens_per_ep_rank=[32794, 32407, 32751, 32742, 33096, 33096, 32463, 32795] avg=32768.0000 max=33096 max_rank=4 min=32407 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65125 max_load_reduction_pct=0.663046 moved_to_replicas=97535 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=46 tokens_per_ep_rank=[32743, 54, 32844, 0, 28, 65536, 65458, 65481] avg=32768.0000 max=65536 max_rank=5 min=0 min_rank=3 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=46 tokens_per_ep_rank=[32743, 32416, 32844, 32440, 32413, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32413 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=97187 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 12, 32745, 65531, 32747, 32795] avg=32768.0000 max=65531 max_rank=5 min=12 min_rank=3 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=47 tokens_per_ep_rank=[32781, 32796, 32737, 32447, 32745, 33096, 32747, 32795] avg=32768.0000 max=33096 max_rank=5 min=32447 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=32435 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=48 tokens_per_ep_rank=[29, 65470, 32767, 65482, 32769, 32768, 32741, 118] avg=32768.0000 max=65482 max_rank=3 min=29 min_rank=0 imbalance_ratio=1.998352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=48 tokens_per_ep_rank=[32415, 33096, 32767, 33096, 32769, 32768, 32741, 32492] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32386 max_load_reduction_pct=0.494579 moved_to_replicas=64760 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=48 total_expected_recv=1571361 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378112 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=48 total_expected_recv=1572291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375296 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=48 total_expected_recv=1574204 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373888 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=48 total_expected_recv=1570070 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378624 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=48 total_expected_recv=1573655 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375936 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=48 total_expected_recv=1572098 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375424 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=48 total_expected_recv=1575595 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372608 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=48 total_expected_recv=1573638 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375168 peak_demand=33097 required_capacity=1.010040 - [2026-08-26 15:01:23] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 737.8 | throughput per GPU (TFLOP/s/GPU): 120.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.118724E+00 | loss scale: 1.0 | grad norm: 2.924 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=31907 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=31907 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32174 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33017 expected_overflow=0 exact_recv_per_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33017 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32543 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32543 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[31867, 17966, 39429, 26298, 40565, 28397, 42672, 34950] avg=32768.0000 max=42672 max_rank=6 min=17966 min_rank=1 imbalance_ratio=1.302246 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[31867, 17966, 39429, 26298, 40565, 28397, 42672, 34950] avg=32768.0000 max=42672 max_rank=6 min=17966 min_rank=1 imbalance_ratio=1.302246 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31701 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9575 max_load_reduction_pct=0.224386 moved_to_replicas=25232 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[31867, 17966, 39429, 26298, 40565, 28397, 42672, 34950] avg=32768.0000 max=42672 max_rank=6 min=17966 min_rank=1 imbalance_ratio=1.302246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[31867, 17966, 39429, 26298, 40565, 28397, 42672, 34950] avg=32768.0000 max=42672 max_rank=6 min=17966 min_rank=1 imbalance_ratio=1.302246 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[31867, 17966, 39429, 26298, 40565, 28397, 42672, 34950] avg=32768.0000 max=42672 max_rank=6 min=17966 min_rank=1 imbalance_ratio=1.302246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31701 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9575 max_load_reduction_pct=0.224386 moved_to_replicas=25232 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31701 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9575 max_load_reduction_pct=0.224386 moved_to_replicas=25232 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31701 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9575 max_load_reduction_pct=0.224386 moved_to_replicas=25232 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[53275, 964, 34904, 32268, 16441, 413, 64410, 59469] avg=32768.0000 max=64410 max_rank=6 min=413 min_rank=5 imbalance_ratio=1.965637 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31701 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9575 max_load_reduction_pct=0.224386 moved_to_replicas=25232 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[31867, 17966, 39429, 26298, 40565, 28397, 42672, 34950] avg=32768.0000 max=42672 max_rank=6 min=17966 min_rank=1 imbalance_ratio=1.302246 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[53275, 964, 34904, 32268, 16441, 413, 64410, 59469] avg=32768.0000 max=64410 max_rank=6 min=413 min_rank=5 imbalance_ratio=1.965637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31727 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31313 max_load_reduction_pct=0.486151 moved_to_replicas=79674 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[53275, 964, 34904, 32268, 16441, 413, 64410, 59469] avg=32768.0000 max=64410 max_rank=6 min=413 min_rank=5 imbalance_ratio=1.965637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[53275, 964, 34904, 32268, 16441, 413, 64410, 59469] avg=32768.0000 max=64410 max_rank=6 min=413 min_rank=5 imbalance_ratio=1.965637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31701 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9575 max_load_reduction_pct=0.224386 moved_to_replicas=25232 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31727 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31313 max_load_reduction_pct=0.486151 moved_to_replicas=79674 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[53275, 964, 34904, 32268, 16441, 413, 64410, 59469] avg=32768.0000 max=64410 max_rank=6 min=413 min_rank=5 imbalance_ratio=1.965637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[31867, 17966, 39429, 26298, 40565, 28397, 42672, 34950] avg=32768.0000 max=42672 max_rank=6 min=17966 min_rank=1 imbalance_ratio=1.302246 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31727 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31313 max_load_reduction_pct=0.486151 moved_to_replicas=79674 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31727 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31313 max_load_reduction_pct=0.486151 moved_to_replicas=79674 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[27859, 13450, 58503, 33355, 33808, 33110, 32675, 29384] avg=32768.0000 max=58503 max_rank=2 min=13450 min_rank=1 imbalance_ratio=1.785370 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31727 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31313 max_load_reduction_pct=0.486151 moved_to_replicas=79674 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[53275, 964, 34904, 32268, 16441, 413, 64410, 59469] avg=32768.0000 max=64410 max_rank=6 min=413 min_rank=5 imbalance_ratio=1.965637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[27859, 13450, 58503, 33355, 33808, 33110, 32675, 29384] avg=32768.0000 max=58503 max_rank=2 min=13450 min_rank=1 imbalance_ratio=1.785370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31701 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9575 max_load_reduction_pct=0.224386 moved_to_replicas=25232 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] avg=32768.0000 max=33097 max_rank=0 min=30893 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25406 max_load_reduction_pct=0.434268 moved_to_replicas=26392 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[27859, 13450, 58503, 33355, 33808, 33110, 32675, 29384] avg=32768.0000 max=58503 max_rank=2 min=13450 min_rank=1 imbalance_ratio=1.785370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[27859, 13450, 58503, 33355, 33808, 33110, 32675, 29384] avg=32768.0000 max=58503 max_rank=2 min=13450 min_rank=1 imbalance_ratio=1.785370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31727 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31313 max_load_reduction_pct=0.486151 moved_to_replicas=79674 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] avg=32768.0000 max=33097 max_rank=0 min=30893 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25406 max_load_reduction_pct=0.434268 moved_to_replicas=26392 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[27859, 13450, 58503, 33355, 33808, 33110, 32675, 29384] avg=32768.0000 max=58503 max_rank=2 min=13450 min_rank=1 imbalance_ratio=1.785370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] avg=32768.0000 max=33097 max_rank=0 min=30893 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25406 max_load_reduction_pct=0.434268 moved_to_replicas=26392 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[53275, 964, 34904, 32268, 16441, 413, 64410, 59469] avg=32768.0000 max=64410 max_rank=6 min=413 min_rank=5 imbalance_ratio=1.965637 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] avg=32768.0000 max=33097 max_rank=0 min=30893 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25406 max_load_reduction_pct=0.434268 moved_to_replicas=26392 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[51334, 6625, 32701, 5783, 4868, 40723, 65753, 54357] avg=32768.0000 max=65753 max_rank=6 min=4868 min_rank=4 imbalance_ratio=2.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] avg=32768.0000 max=33097 max_rank=0 min=30893 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25406 max_load_reduction_pct=0.434268 moved_to_replicas=26392 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[51334, 6625, 32701, 5783, 4868, 40723, 65753, 54357] avg=32768.0000 max=65753 max_rank=6 min=4868 min_rank=4 imbalance_ratio=2.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[27859, 13450, 58503, 33355, 33808, 33110, 32675, 29384] avg=32768.0000 max=58503 max_rank=2 min=13450 min_rank=1 imbalance_ratio=1.785370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31727 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31313 max_load_reduction_pct=0.486151 moved_to_replicas=79674 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] avg=32768.0000 max=33098 max_rank=4 min=30864 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32655 max_load_reduction_pct=0.496631 moved_to_replicas=79783 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[51334, 6625, 32701, 5783, 4868, 40723, 65753, 54357] avg=32768.0000 max=65753 max_rank=6 min=4868 min_rank=4 imbalance_ratio=2.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[51334, 6625, 32701, 5783, 4868, 40723, 65753, 54357] avg=32768.0000 max=65753 max_rank=6 min=4868 min_rank=4 imbalance_ratio=2.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] avg=32768.0000 max=33098 max_rank=4 min=30864 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32655 max_load_reduction_pct=0.496631 moved_to_replicas=79783 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] avg=32768.0000 max=33097 max_rank=0 min=30893 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25406 max_load_reduction_pct=0.434268 moved_to_replicas=26392 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[51334, 6625, 32701, 5783, 4868, 40723, 65753, 54357] avg=32768.0000 max=65753 max_rank=6 min=4868 min_rank=4 imbalance_ratio=2.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] avg=32768.0000 max=33098 max_rank=4 min=30864 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32655 max_load_reduction_pct=0.496631 moved_to_replicas=79783 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] avg=32768.0000 max=33098 max_rank=4 min=30864 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32655 max_load_reduction_pct=0.496631 moved_to_replicas=79783 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65325, 29647, 252, 32730, 32374, 235, 101398, 183] avg=32768.0000 max=101398 max_rank=6 min=183 min_rank=7 imbalance_ratio=3.094421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[27859, 13450, 58503, 33355, 33808, 33110, 32675, 29384] avg=32768.0000 max=58503 max_rank=2 min=13450 min_rank=1 imbalance_ratio=1.785370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] avg=32768.0000 max=33098 max_rank=4 min=30864 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32655 max_load_reduction_pct=0.496631 moved_to_replicas=79783 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65325, 29647, 252, 32730, 32374, 235, 101398, 183] avg=32768.0000 max=101398 max_rank=6 min=183 min_rank=7 imbalance_ratio=3.094421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[51334, 6625, 32701, 5783, 4868, 40723, 65753, 54357] avg=32768.0000 max=65753 max_rank=6 min=4868 min_rank=4 imbalance_ratio=2.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65325, 29647, 252, 32730, 32374, 235, 101398, 183] avg=32768.0000 max=101398 max_rank=6 min=183 min_rank=7 imbalance_ratio=3.094421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=68302 max_load_reduction_pct=0.673603 moved_to_replicas=100531 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] avg=32768.0000 max=33097 max_rank=0 min=30893 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25406 max_load_reduction_pct=0.434268 moved_to_replicas=26392 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65325, 29647, 252, 32730, 32374, 235, 101398, 183] avg=32768.0000 max=101398 max_rank=6 min=183 min_rank=7 imbalance_ratio=3.094421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[31867, 17966, 39429, 26298, 40565, 28397, 42672, 34950] avg=32768.0000 max=42672 max_rank=6 min=17966 min_rank=1 imbalance_ratio=1.302246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=68302 max_load_reduction_pct=0.673603 moved_to_replicas=100531 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] avg=32768.0000 max=33098 max_rank=4 min=30864 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32655 max_load_reduction_pct=0.496631 moved_to_replicas=79783 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=68302 max_load_reduction_pct=0.673603 moved_to_replicas=100531 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65325, 29647, 252, 32730, 32374, 235, 101398, 183] avg=32768.0000 max=101398 max_rank=6 min=183 min_rank=7 imbalance_ratio=3.094421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=68302 max_load_reduction_pct=0.673603 moved_to_replicas=100531 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32786, 22653, 6587, 354, 32835, 232, 64082, 102615] avg=32768.0000 max=102615 max_rank=7 min=232 min_rank=5 imbalance_ratio=3.131561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[31867, 33097, 33096, 33097, 33095, 31701, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31701 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9575 max_load_reduction_pct=0.224386 moved_to_replicas=25232 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[51334, 6625, 32701, 5783, 4868, 40723, 65753, 54357] avg=32768.0000 max=65753 max_rank=6 min=4868 min_rank=4 imbalance_ratio=2.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=68302 max_load_reduction_pct=0.673603 moved_to_replicas=100531 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32786, 22653, 6587, 354, 32835, 232, 64082, 102615] avg=32768.0000 max=102615 max_rank=7 min=232 min_rank=5 imbalance_ratio=3.131561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65325, 29647, 252, 32730, 32374, 235, 101398, 183] avg=32768.0000 max=101398 max_rank=6 min=183 min_rank=7 imbalance_ratio=3.094421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32786, 22653, 6587, 354, 32835, 232, 64082, 102615] avg=32768.0000 max=102615 max_rank=7 min=232 min_rank=5 imbalance_ratio=3.131561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31528 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=69519 max_load_reduction_pct=0.677474 moved_to_replicas=100505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32786, 22653, 6587, 354, 32835, 232, 64082, 102615] avg=32768.0000 max=102615 max_rank=7 min=232 min_rank=5 imbalance_ratio=3.131561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] avg=32768.0000 max=33098 max_rank=4 min=30864 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32655 max_load_reduction_pct=0.496631 moved_to_replicas=79783 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31528 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=69519 max_load_reduction_pct=0.677474 moved_to_replicas=100505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[53275, 964, 34904, 32268, 16441, 413, 64410, 59469] avg=32768.0000 max=64410 max_rank=6 min=413 min_rank=5 imbalance_ratio=1.965637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=68302 max_load_reduction_pct=0.673603 moved_to_replicas=100531 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31528 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=69519 max_load_reduction_pct=0.677474 moved_to_replicas=100505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32786, 22653, 6587, 354, 32835, 232, 64082, 102615] avg=32768.0000 max=102615 max_rank=7 min=232 min_rank=5 imbalance_ratio=3.131561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31528 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=69519 max_load_reduction_pct=0.677474 moved_to_replicas=100505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[122692, 254, 296, 39842, 32728, 61123, 285, 4924] avg=32768.0000 max=122692 max_rank=0 min=254 min_rank=1 imbalance_ratio=3.744263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65325, 29647, 252, 32730, 32374, 235, 101398, 183] avg=32768.0000 max=101398 max_rank=6 min=183 min_rank=7 imbalance_ratio=3.094421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32668, 33096, 32268, 33096, 31727, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31727 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31313 max_load_reduction_pct=0.486151 moved_to_replicas=79674 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31528 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=69519 max_load_reduction_pct=0.677474 moved_to_replicas=100505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[122692, 254, 296, 39842, 32728, 61123, 285, 4924] avg=32768.0000 max=122692 max_rank=0 min=254 min_rank=1 imbalance_ratio=3.744263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32786, 22653, 6587, 354, 32835, 232, 64082, 102615] avg=32768.0000 max=102615 max_rank=7 min=232 min_rank=5 imbalance_ratio=3.131561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] avg=32768.0000 max=33097 max_rank=3 min=31590 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=89595 max_load_reduction_pct=0.730243 moved_to_replicas=124369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[122692, 254, 296, 39842, 32728, 61123, 285, 4924] avg=32768.0000 max=122692 max_rank=0 min=254 min_rank=1 imbalance_ratio=3.744263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[122692, 254, 296, 39842, 32728, 61123, 285, 4924] avg=32768.0000 max=122692 max_rank=0 min=254 min_rank=1 imbalance_ratio=3.744263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=68302 max_load_reduction_pct=0.673603 moved_to_replicas=100531 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] avg=32768.0000 max=33097 max_rank=3 min=31590 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=89595 max_load_reduction_pct=0.730243 moved_to_replicas=124369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31528 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=69519 max_load_reduction_pct=0.677474 moved_to_replicas=100505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[27859, 13450, 58503, 33355, 33808, 33110, 32675, 29384] avg=32768.0000 max=58503 max_rank=2 min=13450 min_rank=1 imbalance_ratio=1.785370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[122692, 254, 296, 39842, 32728, 61123, 285, 4924] avg=32768.0000 max=122692 max_rank=0 min=254 min_rank=1 imbalance_ratio=3.744263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] avg=32768.0000 max=33097 max_rank=3 min=31590 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=89595 max_load_reduction_pct=0.730243 moved_to_replicas=124369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] avg=32768.0000 max=33097 max_rank=3 min=31590 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=89595 max_load_reduction_pct=0.730243 moved_to_replicas=124369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65279, 19552, 173, 32664, 33015, 65619, 19938, 25904] avg=32768.0000 max=65619 max_rank=5 min=173 min_rank=2 imbalance_ratio=2.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32786, 22653, 6587, 354, 32835, 232, 64082, 102615] avg=32768.0000 max=102615 max_rank=7 min=232 min_rank=5 imbalance_ratio=3.131561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 33096, 32675, 30893] avg=32768.0000 max=33097 max_rank=0 min=30893 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25406 max_load_reduction_pct=0.434268 moved_to_replicas=26392 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] avg=32768.0000 max=33097 max_rank=3 min=31590 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=89595 max_load_reduction_pct=0.730243 moved_to_replicas=124369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65279, 19552, 173, 32664, 33015, 65619, 19938, 25904] avg=32768.0000 max=65619 max_rank=5 min=173 min_rank=2 imbalance_ratio=2.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[122692, 254, 296, 39842, 32728, 61123, 285, 4924] avg=32768.0000 max=122692 max_rank=0 min=254 min_rank=1 imbalance_ratio=3.744263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] avg=32768.0000 max=33096 max_rank=0 min=31385 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32523 max_load_reduction_pct=0.495634 moved_to_replicas=64706 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65279, 19552, 173, 32664, 33015, 65619, 19938, 25904] avg=32768.0000 max=65619 max_rank=5 min=173 min_rank=2 imbalance_ratio=2.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65279, 19552, 173, 32664, 33015, 65619, 19938, 25904] avg=32768.0000 max=65619 max_rank=5 min=173 min_rank=2 imbalance_ratio=2.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31528 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=69519 max_load_reduction_pct=0.677474 moved_to_replicas=100505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] avg=32768.0000 max=33096 max_rank=0 min=31385 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32523 max_load_reduction_pct=0.495634 moved_to_replicas=64706 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] avg=32768.0000 max=33097 max_rank=3 min=31590 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=89595 max_load_reduction_pct=0.730243 moved_to_replicas=124369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[51334, 6625, 32701, 5783, 4868, 40723, 65753, 54357] avg=32768.0000 max=65753 max_rank=6 min=4868 min_rank=4 imbalance_ratio=2.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] avg=32768.0000 max=33096 max_rank=0 min=31385 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32523 max_load_reduction_pct=0.495634 moved_to_replicas=64706 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] avg=32768.0000 max=33096 max_rank=0 min=31385 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32523 max_load_reduction_pct=0.495634 moved_to_replicas=64706 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65279, 19552, 173, 32664, 33015, 65619, 19938, 25904] avg=32768.0000 max=65619 max_rank=5 min=173 min_rank=2 imbalance_ratio=2.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[644, 32654, 32875, 21452, 97149, 167, 44446, 32757] avg=32768.0000 max=97149 max_rank=4 min=167 min_rank=5 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[122692, 254, 296, 39842, 32728, 61123, 285, 4924] avg=32768.0000 max=122692 max_rank=0 min=254 min_rank=1 imbalance_ratio=3.744263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[33097, 33096, 32701, 30864, 33098, 33097, 33096, 33095] avg=32768.0000 max=33098 max_rank=4 min=30864 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32655 max_load_reduction_pct=0.496631 moved_to_replicas=79783 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[644, 32654, 32875, 21452, 97149, 167, 44446, 32757] avg=32768.0000 max=97149 max_rank=4 min=167 min_rank=5 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65279, 19552, 173, 32664, 33015, 65619, 19938, 25904] avg=32768.0000 max=65619 max_rank=5 min=173 min_rank=2 imbalance_ratio=2.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] avg=32768.0000 max=33096 max_rank=0 min=31385 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32523 max_load_reduction_pct=0.495634 moved_to_replicas=64706 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] avg=32768.0000 max=33096 max_rank=4 min=32059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=75403 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[644, 32654, 32875, 21452, 97149, 167, 44446, 32757] avg=32768.0000 max=97149 max_rank=4 min=167 min_rank=5 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[644, 32654, 32875, 21452, 97149, 167, 44446, 32757] avg=32768.0000 max=97149 max_rank=4 min=167 min_rank=5 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] avg=32768.0000 max=33097 max_rank=3 min=31590 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=89595 max_load_reduction_pct=0.730243 moved_to_replicas=124369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] avg=32768.0000 max=33096 max_rank=4 min=32059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=75403 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] avg=32768.0000 max=33096 max_rank=0 min=31385 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32523 max_load_reduction_pct=0.495634 moved_to_replicas=64706 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] avg=32768.0000 max=33096 max_rank=4 min=32059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=75403 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65325, 29647, 252, 32730, 32374, 235, 101398, 183] avg=32768.0000 max=101398 max_rank=6 min=183 min_rank=7 imbalance_ratio=3.094421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] avg=32768.0000 max=33096 max_rank=4 min=32059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=75403 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[644, 32654, 32875, 21452, 97149, 167, 44446, 32757] avg=32768.0000 max=97149 max_rank=4 min=167 min_rank=5 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[44387, 32741, 61, 12496, 65143, 12256, 32784, 62276] avg=32768.0000 max=65143 max_rank=4 min=61 min_rank=2 imbalance_ratio=1.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65279, 19552, 173, 32664, 33015, 65619, 19938, 25904] avg=32768.0000 max=65619 max_rank=5 min=173 min_rank=2 imbalance_ratio=2.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32182, 33096, 32730, 32374, 32778, 33096, 32792] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=68302 max_load_reduction_pct=0.673603 moved_to_replicas=100531 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[44387, 32741, 61, 12496, 65143, 12256, 32784, 62276] avg=32768.0000 max=65143 max_rank=4 min=61 min_rank=2 imbalance_ratio=1.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] avg=32768.0000 max=33097 max_rank=0 min=32108 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32046 max_load_reduction_pct=0.491933 moved_to_replicas=72518 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] avg=32768.0000 max=33096 max_rank=4 min=32059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=75403 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[644, 32654, 32875, 21452, 97149, 167, 44446, 32757] avg=32768.0000 max=97149 max_rank=4 min=167 min_rank=5 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[44387, 32741, 61, 12496, 65143, 12256, 32784, 62276] avg=32768.0000 max=65143 max_rank=4 min=61 min_rank=2 imbalance_ratio=1.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[44387, 32741, 61, 12496, 65143, 12256, 32784, 62276] avg=32768.0000 max=65143 max_rank=4 min=61 min_rank=2 imbalance_ratio=1.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] avg=32768.0000 max=33096 max_rank=0 min=31385 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32523 max_load_reduction_pct=0.495634 moved_to_replicas=64706 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] avg=32768.0000 max=33097 max_rank=0 min=32108 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32046 max_load_reduction_pct=0.491933 moved_to_replicas=72518 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] avg=32768.0000 max=33096 max_rank=4 min=32059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=75403 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] avg=32768.0000 max=33097 max_rank=0 min=32108 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32046 max_load_reduction_pct=0.491933 moved_to_replicas=72518 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32786, 22653, 6587, 354, 32835, 232, 64082, 102615] avg=32768.0000 max=102615 max_rank=7 min=232 min_rank=5 imbalance_ratio=3.131561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] avg=32768.0000 max=33097 max_rank=0 min=32108 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32046 max_load_reduction_pct=0.491933 moved_to_replicas=72518 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[32658, 27, 135, 130609, 20160, 40014, 5467, 33074] avg=32768.0000 max=130609 max_rank=3 min=27 min_rank=1 imbalance_ratio=3.985870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[44387, 32741, 61, 12496, 65143, 12256, 32784, 62276] avg=32768.0000 max=65143 max_rank=4 min=61 min_rank=2 imbalance_ratio=1.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[644, 32654, 32875, 21452, 97149, 167, 44446, 32757] avg=32768.0000 max=97149 max_rank=4 min=167 min_rank=5 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32786, 31528, 33096, 32899, 32835, 32808, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31528 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=69519 max_load_reduction_pct=0.677474 moved_to_replicas=100505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] avg=32768.0000 max=33096 max_rank=3 min=31687 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97513 max_load_reduction_pct=0.746602 moved_to_replicas=104431 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[32658, 27, 135, 130609, 20160, 40014, 5467, 33074] avg=32768.0000 max=130609 max_rank=3 min=27 min_rank=1 imbalance_ratio=3.985870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] avg=32768.0000 max=33097 max_rank=0 min=32108 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32046 max_load_reduction_pct=0.491933 moved_to_replicas=72518 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[32658, 27, 135, 130609, 20160, 40014, 5467, 33074] avg=32768.0000 max=130609 max_rank=3 min=27 min_rank=1 imbalance_ratio=3.985870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[44387, 32741, 61, 12496, 65143, 12256, 32784, 62276] avg=32768.0000 max=65143 max_rank=4 min=61 min_rank=2 imbalance_ratio=1.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[32658, 27, 135, 130609, 20160, 40014, 5467, 33074] avg=32768.0000 max=130609 max_rank=3 min=27 min_rank=1 imbalance_ratio=3.985870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] avg=32768.0000 max=33096 max_rank=4 min=32059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=75403 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] avg=32768.0000 max=33096 max_rank=3 min=31687 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97513 max_load_reduction_pct=0.746602 moved_to_replicas=104431 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] avg=32768.0000 max=33096 max_rank=3 min=31687 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97513 max_load_reduction_pct=0.746602 moved_to_replicas=104431 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] avg=32768.0000 max=33097 max_rank=0 min=32108 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32046 max_load_reduction_pct=0.491933 moved_to_replicas=72518 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] avg=32768.0000 max=33096 max_rank=3 min=31687 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97513 max_load_reduction_pct=0.746602 moved_to_replicas=104431 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65904, 98027, 122, 53488, 287, 11413, 133, 32770] avg=32768.0000 max=98027 max_rank=1 min=122 min_rank=2 imbalance_ratio=2.991547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[122692, 254, 296, 39842, 32728, 61123, 285, 4924] avg=32768.0000 max=122692 max_rank=0 min=254 min_rank=1 imbalance_ratio=3.744263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[32658, 27, 135, 130609, 20160, 40014, 5467, 33074] avg=32768.0000 max=130609 max_rank=3 min=27 min_rank=1 imbalance_ratio=3.985870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[44387, 32741, 61, 12496, 65143, 12256, 32784, 62276] avg=32768.0000 max=65143 max_rank=4 min=61 min_rank=2 imbalance_ratio=1.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] avg=32768.0000 max=33096 max_rank=0 min=31976 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64931 max_load_reduction_pct=0.662379 moved_to_replicas=118131 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32803, 31590, 33097, 32728, 33096, 32783, 32951] avg=32768.0000 max=33097 max_rank=3 min=31590 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=89595 max_load_reduction_pct=0.730243 moved_to_replicas=124369 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65904, 98027, 122, 53488, 287, 11413, 133, 32770] avg=32768.0000 max=98027 max_rank=1 min=122 min_rank=2 imbalance_ratio=2.991547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65904, 98027, 122, 53488, 287, 11413, 133, 32770] avg=32768.0000 max=98027 max_rank=1 min=122 min_rank=2 imbalance_ratio=2.991547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] avg=32768.0000 max=33096 max_rank=3 min=31687 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97513 max_load_reduction_pct=0.746602 moved_to_replicas=104431 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[32658, 27, 135, 130609, 20160, 40014, 5467, 33074] avg=32768.0000 max=130609 max_rank=3 min=27 min_rank=1 imbalance_ratio=3.985870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65904, 98027, 122, 53488, 287, 11413, 133, 32770] avg=32768.0000 max=98027 max_rank=1 min=122 min_rank=2 imbalance_ratio=2.991547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] avg=32768.0000 max=33097 max_rank=0 min=32108 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32046 max_load_reduction_pct=0.491933 moved_to_replicas=72518 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] avg=32768.0000 max=33096 max_rank=0 min=31976 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64931 max_load_reduction_pct=0.662379 moved_to_replicas=118131 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] avg=32768.0000 max=33096 max_rank=0 min=31976 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64931 max_load_reduction_pct=0.662379 moved_to_replicas=118131 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] avg=32768.0000 max=33096 max_rank=3 min=31687 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97513 max_load_reduction_pct=0.746602 moved_to_replicas=104431 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] avg=32768.0000 max=33096 max_rank=0 min=31976 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64931 max_load_reduction_pct=0.662379 moved_to_replicas=118131 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[151, 65388, 32755, 34078, 65343, 74, 526, 63829] avg=32768.0000 max=65388 max_rank=1 min=74 min_rank=5 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65279, 19552, 173, 32664, 33015, 65619, 19938, 25904] avg=32768.0000 max=65619 max_rank=5 min=173 min_rank=2 imbalance_ratio=2.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65904, 98027, 122, 53488, 287, 11413, 133, 32770] avg=32768.0000 max=98027 max_rank=1 min=122 min_rank=2 imbalance_ratio=2.991547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[32658, 27, 135, 130609, 20160, 40014, 5467, 33074] avg=32768.0000 max=130609 max_rank=3 min=27 min_rank=1 imbalance_ratio=3.985870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] avg=32768.0000 max=33096 max_rank=1 min=32241 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32292 max_load_reduction_pct=0.493852 moved_to_replicas=96254 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[151, 65388, 32755, 34078, 65343, 74, 526, 63829] avg=32768.0000 max=65388 max_rank=1 min=74 min_rank=5 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[151, 65388, 32755, 34078, 65343, 74, 526, 63829] avg=32768.0000 max=65388 max_rank=1 min=74 min_rank=5 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[33096, 33096, 32696, 32664, 33015, 33096, 33096, 31385] avg=32768.0000 max=33096 max_rank=0 min=31385 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32523 max_load_reduction_pct=0.495634 moved_to_replicas=64706 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65904, 98027, 122, 53488, 287, 11413, 133, 32770] avg=32768.0000 max=98027 max_rank=1 min=122 min_rank=2 imbalance_ratio=2.991547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] avg=32768.0000 max=33096 max_rank=0 min=31976 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64931 max_load_reduction_pct=0.662379 moved_to_replicas=118131 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[151, 65388, 32755, 34078, 65343, 74, 526, 63829] avg=32768.0000 max=65388 max_rank=1 min=74 min_rank=5 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] avg=32768.0000 max=33096 max_rank=3 min=31687 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97513 max_load_reduction_pct=0.746602 moved_to_replicas=104431 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] avg=32768.0000 max=33096 max_rank=1 min=32241 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32292 max_load_reduction_pct=0.493852 moved_to_replicas=96254 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] avg=32768.0000 max=33096 max_rank=1 min=32241 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32292 max_load_reduction_pct=0.493852 moved_to_replicas=96254 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] avg=32768.0000 max=33096 max_rank=0 min=31976 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64931 max_load_reduction_pct=0.662379 moved_to_replicas=118131 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] avg=32768.0000 max=33096 max_rank=1 min=32241 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32292 max_load_reduction_pct=0.493852 moved_to_replicas=96254 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[17158, 32946, 44198, 37042, 133, 65266, 65354, 47] avg=32768.0000 max=65354 max_rank=6 min=47 min_rank=7 imbalance_ratio=1.994446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[151, 65388, 32755, 34078, 65343, 74, 526, 63829] avg=32768.0000 max=65388 max_rank=1 min=74 min_rank=5 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[644, 32654, 32875, 21452, 97149, 167, 44446, 32757] avg=32768.0000 max=97149 max_rank=4 min=167 min_rank=5 imbalance_ratio=2.964752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65904, 98027, 122, 53488, 287, 11413, 133, 32770] avg=32768.0000 max=98027 max_rank=1 min=122 min_rank=2 imbalance_ratio=2.991547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] avg=32768.0000 max=33096 max_rank=2 min=32206 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32258 max_load_reduction_pct=0.493589 moved_to_replicas=79476 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[17158, 32946, 44198, 37042, 133, 65266, 65354, 47] avg=32768.0000 max=65354 max_rank=6 min=47 min_rank=7 imbalance_ratio=1.994446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[17158, 32946, 44198, 37042, 133, 65266, 65354, 47] avg=32768.0000 max=65354 max_rank=6 min=47 min_rank=7 imbalance_ratio=1.994446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] avg=32768.0000 max=33096 max_rank=1 min=32241 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32292 max_load_reduction_pct=0.493852 moved_to_replicas=96254 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[151, 65388, 32755, 34078, 65343, 74, 526, 63829] avg=32768.0000 max=65388 max_rank=1 min=74 min_rank=5 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[17158, 32946, 44198, 37042, 133, 65266, 65354, 47] avg=32768.0000 max=65354 max_rank=6 min=47 min_rank=7 imbalance_ratio=1.994446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32059, 32654, 32875, 32802, 33096, 32805, 33096, 32757] avg=32768.0000 max=33096 max_rank=4 min=32059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64053 max_load_reduction_pct=0.659327 moved_to_replicas=75403 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] avg=32768.0000 max=33096 max_rank=0 min=31976 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64931 max_load_reduction_pct=0.662379 moved_to_replicas=118131 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] avg=32768.0000 max=33096 max_rank=2 min=32206 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32258 max_load_reduction_pct=0.493589 moved_to_replicas=79476 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] avg=32768.0000 max=33096 max_rank=2 min=32206 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32258 max_load_reduction_pct=0.493589 moved_to_replicas=79476 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[26013, 104799, 94, 32819, 47, 32767, 65313, 292] avg=32768.0000 max=104799 max_rank=1 min=47 min_rank=4 imbalance_ratio=3.198212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] avg=32768.0000 max=33096 max_rank=1 min=32241 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32292 max_load_reduction_pct=0.493852 moved_to_replicas=96254 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] avg=32768.0000 max=33096 max_rank=2 min=32206 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32258 max_load_reduction_pct=0.493589 moved_to_replicas=79476 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[17158, 32946, 44198, 37042, 133, 65266, 65354, 47] avg=32768.0000 max=65354 max_rank=6 min=47 min_rank=7 imbalance_ratio=1.994446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31849 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=71702 max_load_reduction_pct=0.684186 moved_to_replicas=103920 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[44387, 32741, 61, 12496, 65143, 12256, 32784, 62276] avg=32768.0000 max=65143 max_rank=4 min=61 min_rank=2 imbalance_ratio=1.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[151, 65388, 32755, 34078, 65343, 74, 526, 63829] avg=32768.0000 max=65388 max_rank=1 min=74 min_rank=5 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[26013, 104799, 94, 32819, 47, 32767, 65313, 292] avg=32768.0000 max=104799 max_rank=1 min=47 min_rank=4 imbalance_ratio=3.198212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[26013, 104799, 94, 32819, 47, 32767, 65313, 292] avg=32768.0000 max=104799 max_rank=1 min=47 min_rank=4 imbalance_ratio=3.198212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] avg=32768.0000 max=33096 max_rank=2 min=32206 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32258 max_load_reduction_pct=0.493589 moved_to_replicas=79476 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[26013, 104799, 94, 32819, 47, 32767, 65313, 292] avg=32768.0000 max=104799 max_rank=1 min=47 min_rank=4 imbalance_ratio=3.198212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[17158, 32946, 44198, 37042, 133, 65266, 65354, 47] avg=32768.0000 max=65354 max_rank=6 min=47 min_rank=7 imbalance_ratio=1.994446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[33097, 32741, 32108, 32127, 33096, 33095, 32784, 33096] avg=32768.0000 max=33097 max_rank=0 min=32108 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32046 max_load_reduction_pct=0.491933 moved_to_replicas=72518 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] avg=32768.0000 max=33096 max_rank=1 min=32241 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32292 max_load_reduction_pct=0.493852 moved_to_replicas=96254 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31849 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=71702 max_load_reduction_pct=0.684186 moved_to_replicas=103920 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32653, 82, 32924, 16008, 81858, 33057, 32886, 32676] avg=32768.0000 max=81858 max_rank=4 min=82 min_rank=1 imbalance_ratio=2.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31849 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=71702 max_load_reduction_pct=0.684186 moved_to_replicas=103920 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31849 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=71702 max_load_reduction_pct=0.684186 moved_to_replicas=103920 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] avg=32768.0000 max=33096 max_rank=2 min=32206 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32258 max_load_reduction_pct=0.493589 moved_to_replicas=79476 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[26013, 104799, 94, 32819, 47, 32767, 65313, 292] avg=32768.0000 max=104799 max_rank=1 min=47 min_rank=4 imbalance_ratio=3.198212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[32658, 27, 135, 130609, 20160, 40014, 5467, 33074] avg=32768.0000 max=130609 max_rank=3 min=27 min_rank=1 imbalance_ratio=3.985870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] avg=32768.0000 max=33095 max_rank=4 min=32137 min_rank=3 imbalance_ratio=1.009979 max_load_reduction=48763 max_load_reduction_pct=0.595702 moved_to_replicas=48762 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[17158, 32946, 44198, 37042, 133, 65266, 65354, 47] avg=32768.0000 max=65354 max_rank=6 min=47 min_rank=7 imbalance_ratio=1.994446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32653, 82, 32924, 16008, 81858, 33057, 32886, 32676] avg=32768.0000 max=81858 max_rank=4 min=82 min_rank=1 imbalance_ratio=2.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32653, 82, 32924, 16008, 81858, 33057, 32886, 32676] avg=32768.0000 max=81858 max_rank=4 min=82 min_rank=1 imbalance_ratio=2.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32653, 82, 32924, 16008, 81858, 33057, 32886, 32676] avg=32768.0000 max=81858 max_rank=4 min=82 min_rank=1 imbalance_ratio=2.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31849 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=71702 max_load_reduction_pct=0.684186 moved_to_replicas=103920 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[26013, 104799, 94, 32819, 47, 32767, 65313, 292] avg=32768.0000 max=104799 max_rank=1 min=47 min_rank=4 imbalance_ratio=3.198212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[32658, 32667, 32771, 33096, 31687, 33096, 33095, 33074] avg=32768.0000 max=33096 max_rank=3 min=31687 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97513 max_load_reduction_pct=0.746602 moved_to_replicas=104431 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] avg=32768.0000 max=33096 max_rank=2 min=32206 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32258 max_load_reduction_pct=0.493589 moved_to_replicas=79476 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] avg=32768.0000 max=33095 max_rank=4 min=32137 min_rank=3 imbalance_ratio=1.009979 max_load_reduction=48763 max_load_reduction_pct=0.595702 moved_to_replicas=48762 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[32065, 445, 32657, 32773, 66691, 64522, 32781, 210] avg=32768.0000 max=66691 max_rank=4 min=210 min_rank=7 imbalance_ratio=2.035248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] avg=32768.0000 max=33095 max_rank=4 min=32137 min_rank=3 imbalance_ratio=1.009979 max_load_reduction=48763 max_load_reduction_pct=0.595702 moved_to_replicas=48762 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] avg=32768.0000 max=33095 max_rank=4 min=32137 min_rank=3 imbalance_ratio=1.009979 max_load_reduction=48763 max_load_reduction_pct=0.595702 moved_to_replicas=48762 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31849 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=71702 max_load_reduction_pct=0.684186 moved_to_replicas=103920 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32653, 82, 32924, 16008, 81858, 33057, 32886, 32676] avg=32768.0000 max=81858 max_rank=4 min=82 min_rank=1 imbalance_ratio=2.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65904, 98027, 122, 53488, 287, 11413, 133, 32770] avg=32768.0000 max=98027 max_rank=1 min=122 min_rank=2 imbalance_ratio=2.991547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] avg=32768.0000 max=33096 max_rank=4 min=32065 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33595 max_load_reduction_pct=0.503741 moved_to_replicas=65021 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[26013, 104799, 94, 32819, 47, 32767, 65313, 292] avg=32768.0000 max=104799 max_rank=1 min=47 min_rank=4 imbalance_ratio=3.198212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[32065, 445, 32657, 32773, 66691, 64522, 32781, 210] avg=32768.0000 max=66691 max_rank=4 min=210 min_rank=7 imbalance_ratio=2.035248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[32065, 445, 32657, 32773, 66691, 64522, 32781, 210] avg=32768.0000 max=66691 max_rank=4 min=210 min_rank=7 imbalance_ratio=2.035248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[32065, 445, 32657, 32773, 66691, 64522, 32781, 210] avg=32768.0000 max=66691 max_rank=4 min=210 min_rank=7 imbalance_ratio=2.035248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] avg=32768.0000 max=33095 max_rank=4 min=32137 min_rank=3 imbalance_ratio=1.009979 max_load_reduction=48763 max_load_reduction_pct=0.595702 moved_to_replicas=48762 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32653, 82, 32924, 16008, 81858, 33057, 32886, 32676] avg=32768.0000 max=81858 max_rank=4 min=82 min_rank=1 imbalance_ratio=2.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[33096, 33096, 32757, 33096, 32924, 31976, 32429, 32770] avg=32768.0000 max=33096 max_rank=0 min=31976 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64931 max_load_reduction_pct=0.662379 moved_to_replicas=118131 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31849 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=71702 max_load_reduction_pct=0.684186 moved_to_replicas=103920 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] avg=32768.0000 max=33096 max_rank=4 min=32065 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33595 max_load_reduction_pct=0.503741 moved_to_replicas=65021 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[32666, 757, 32891, 12279, 50350, 32832, 32479, 67890] avg=32768.0000 max=67890 max_rank=7 min=757 min_rank=1 imbalance_ratio=2.071838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] avg=32768.0000 max=33096 max_rank=4 min=32065 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33595 max_load_reduction_pct=0.503741 moved_to_replicas=65021 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] avg=32768.0000 max=33096 max_rank=4 min=32065 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33595 max_load_reduction_pct=0.503741 moved_to_replicas=65021 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] avg=32768.0000 max=33095 max_rank=4 min=32137 min_rank=3 imbalance_ratio=1.009979 max_load_reduction=48763 max_load_reduction_pct=0.595702 moved_to_replicas=48762 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[32065, 445, 32657, 32773, 66691, 64522, 32781, 210] avg=32768.0000 max=66691 max_rank=4 min=210 min_rank=7 imbalance_ratio=2.035248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] avg=32768.0000 max=33097 max_rank=4 min=31987 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34793 max_load_reduction_pct=0.512491 moved_to_replicas=52048 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[151, 65388, 32755, 34078, 65343, 74, 526, 63829] avg=32768.0000 max=65388 max_rank=1 min=74 min_rank=5 imbalance_ratio=1.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[32666, 757, 32891, 12279, 50350, 32832, 32479, 67890] avg=32768.0000 max=67890 max_rank=7 min=757 min_rank=1 imbalance_ratio=2.071838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32653, 82, 32924, 16008, 81858, 33057, 32886, 32676] avg=32768.0000 max=81858 max_rank=4 min=82 min_rank=1 imbalance_ratio=2.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[32666, 757, 32891, 12279, 50350, 32832, 32479, 67890] avg=32768.0000 max=67890 max_rank=7 min=757 min_rank=1 imbalance_ratio=2.071838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[32666, 757, 32891, 12279, 50350, 32832, 32479, 67890] avg=32768.0000 max=67890 max_rank=7 min=757 min_rank=1 imbalance_ratio=2.071838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] avg=32768.0000 max=33096 max_rank=4 min=32065 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33595 max_load_reduction_pct=0.503741 moved_to_replicas=65021 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[32065, 445, 32657, 32773, 66691, 64522, 32781, 210] avg=32768.0000 max=66691 max_rank=4 min=210 min_rank=7 imbalance_ratio=2.035248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32398, 33096, 32755, 33096, 33096, 32366, 32241, 33096] avg=32768.0000 max=33096 max_rank=1 min=32241 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32292 max_load_reduction_pct=0.493852 moved_to_replicas=96254 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] avg=32768.0000 max=33097 max_rank=4 min=31987 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34793 max_load_reduction_pct=0.512491 moved_to_replicas=52048 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] avg=32768.0000 max=33095 max_rank=4 min=32137 min_rank=3 imbalance_ratio=1.009979 max_load_reduction=48763 max_load_reduction_pct=0.595702 moved_to_replicas=48762 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[32905, 64000, 13894, 268, 1126, 64897, 19676, 65378] avg=32768.0000 max=65378 max_rank=7 min=268 min_rank=3 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] avg=32768.0000 max=33097 max_rank=4 min=31987 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34793 max_load_reduction_pct=0.512491 moved_to_replicas=52048 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] avg=32768.0000 max=33097 max_rank=4 min=31987 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34793 max_load_reduction_pct=0.512491 moved_to_replicas=52048 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] avg=32768.0000 max=33096 max_rank=4 min=32065 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33595 max_load_reduction_pct=0.503741 moved_to_replicas=65021 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[32666, 757, 32891, 12279, 50350, 32832, 32479, 67890] avg=32768.0000 max=67890 max_rank=7 min=757 min_rank=1 imbalance_ratio=2.071838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] avg=32768.0000 max=33096 max_rank=1 min=31378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=94987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[17158, 32946, 44198, 37042, 133, 65266, 65354, 47] avg=32768.0000 max=65354 max_rank=6 min=47 min_rank=7 imbalance_ratio=1.994446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[32905, 64000, 13894, 268, 1126, 64897, 19676, 65378] avg=32768.0000 max=65378 max_rank=7 min=268 min_rank=3 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[32065, 445, 32657, 32773, 66691, 64522, 32781, 210] avg=32768.0000 max=66691 max_rank=4 min=210 min_rank=7 imbalance_ratio=2.035248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[32905, 64000, 13894, 268, 1126, 64897, 19676, 65378] avg=32768.0000 max=65378 max_rank=7 min=268 min_rank=3 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[32905, 64000, 13894, 268, 1126, 64897, 19676, 65378] avg=32768.0000 max=65378 max_rank=7 min=268 min_rank=3 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] avg=32768.0000 max=33097 max_rank=4 min=31987 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34793 max_load_reduction_pct=0.512491 moved_to_replicas=52048 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[32666, 757, 32891, 12279, 50350, 32832, 32479, 67890] avg=32768.0000 max=67890 max_rank=7 min=757 min_rank=1 imbalance_ratio=2.071838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32206, 32946, 33096, 33096, 32303, 33096, 33096, 32305] avg=32768.0000 max=33096 max_rank=2 min=32206 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32258 max_load_reduction_pct=0.493589 moved_to_replicas=79476 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] avg=32768.0000 max=33096 max_rank=1 min=31378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=94987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[22147, 59841, 48209, 32667, 271, 32580, 33521, 32908] avg=32768.0000 max=59841 max_rank=1 min=271 min_rank=4 imbalance_ratio=1.826202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] avg=32768.0000 max=33096 max_rank=4 min=32065 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33595 max_load_reduction_pct=0.503741 moved_to_replicas=65021 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] avg=32768.0000 max=33096 max_rank=1 min=31378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=94987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] avg=32768.0000 max=33096 max_rank=1 min=31378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=94987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] avg=32768.0000 max=33097 max_rank=4 min=31987 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34793 max_load_reduction_pct=0.512491 moved_to_replicas=52048 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[32905, 64000, 13894, 268, 1126, 64897, 19676, 65378] avg=32768.0000 max=65378 max_rank=7 min=268 min_rank=3 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] avg=32768.0000 max=33097 max_rank=1 min=31604 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=26744 max_load_reduction_pct=0.446918 moved_to_replicas=42283 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[26013, 104799, 94, 32819, 47, 32767, 65313, 292] avg=32768.0000 max=104799 max_rank=1 min=47 min_rank=4 imbalance_ratio=3.198212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[22147, 59841, 48209, 32667, 271, 32580, 33521, 32908] avg=32768.0000 max=59841 max_rank=1 min=271 min_rank=4 imbalance_ratio=1.826202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[32666, 757, 32891, 12279, 50350, 32832, 32479, 67890] avg=32768.0000 max=67890 max_rank=7 min=757 min_rank=1 imbalance_ratio=2.071838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[22147, 59841, 48209, 32667, 271, 32580, 33521, 32908] avg=32768.0000 max=59841 max_rank=1 min=271 min_rank=4 imbalance_ratio=1.826202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] avg=32768.0000 max=33096 max_rank=1 min=31378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=94987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[22147, 59841, 48209, 32667, 271, 32580, 33521, 32908] avg=32768.0000 max=59841 max_rank=1 min=271 min_rank=4 imbalance_ratio=1.826202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[32905, 64000, 13894, 268, 1126, 64897, 19676, 65378] avg=32768.0000 max=65378 max_rank=7 min=268 min_rank=3 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 65369, 65380, 267, 32618, 32733, 347] avg=32768.0000 max=65380 max_rank=3 min=267 min_rank=4 imbalance_ratio=1.995239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] avg=32768.0000 max=33097 max_rank=1 min=31604 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=26744 max_load_reduction_pct=0.446918 moved_to_replicas=42283 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[31849, 33096, 32733, 32819, 32687, 32767, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31849 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=71702 max_load_reduction_pct=0.684186 moved_to_replicas=103920 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] avg=32768.0000 max=33097 max_rank=4 min=31987 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34793 max_load_reduction_pct=0.512491 moved_to_replicas=52048 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] avg=32768.0000 max=33097 max_rank=1 min=31604 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=26744 max_load_reduction_pct=0.446918 moved_to_replicas=42283 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] avg=32768.0000 max=33097 max_rank=1 min=31604 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=26744 max_load_reduction_pct=0.446918 moved_to_replicas=42283 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] avg=32768.0000 max=33096 max_rank=1 min=31378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=94987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[22147, 59841, 48209, 32667, 271, 32580, 33521, 32908] avg=32768.0000 max=59841 max_rank=1 min=271 min_rank=4 imbalance_ratio=1.826202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] avg=32768.0000 max=33096 max_rank=2 min=32551 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32284 max_load_reduction_pct=0.493790 moved_to_replicas=64557 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 65369, 65380, 267, 32618, 32733, 347] avg=32768.0000 max=65380 max_rank=3 min=267 min_rank=4 imbalance_ratio=1.995239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32653, 82, 32924, 16008, 81858, 33057, 32886, 32676] avg=32768.0000 max=81858 max_rank=4 min=82 min_rank=1 imbalance_ratio=2.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] avg=32768.0000 max=33097 max_rank=1 min=31604 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=26744 max_load_reduction_pct=0.446918 moved_to_replicas=42283 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[22147, 59841, 48209, 32667, 271, 32580, 33521, 32908] avg=32768.0000 max=59841 max_rank=1 min=271 min_rank=4 imbalance_ratio=1.826202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 65369, 65380, 267, 32618, 32733, 347] avg=32768.0000 max=65380 max_rank=3 min=267 min_rank=4 imbalance_ratio=1.995239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 65369, 65380, 267, 32618, 32733, 347] avg=32768.0000 max=65380 max_rank=3 min=267 min_rank=4 imbalance_ratio=1.995239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[32905, 64000, 13894, 268, 1126, 64897, 19676, 65378] avg=32768.0000 max=65378 max_rank=7 min=268 min_rank=3 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[62793, 66887, 32629, 27043, 157, 39728, 32667, 240] avg=32768.0000 max=66887 max_rank=1 min=157 min_rank=4 imbalance_ratio=2.041229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] avg=32768.0000 max=33096 max_rank=2 min=32551 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32284 max_load_reduction_pct=0.493790 moved_to_replicas=64557 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32653, 32716, 32924, 32137, 33095, 33057, 32886, 32676] avg=32768.0000 max=33095 max_rank=4 min=32137 min_rank=3 imbalance_ratio=1.009979 max_load_reduction=48763 max_load_reduction_pct=0.595702 moved_to_replicas=48762 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] avg=32768.0000 max=33097 max_rank=1 min=31604 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=26744 max_load_reduction_pct=0.446918 moved_to_replicas=42283 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] avg=32768.0000 max=33096 max_rank=2 min=32551 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32284 max_load_reduction_pct=0.493790 moved_to_replicas=64557 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] avg=32768.0000 max=33096 max_rank=2 min=32551 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32284 max_load_reduction_pct=0.493790 moved_to_replicas=64557 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] avg=32768.0000 max=33096 max_rank=1 min=31378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=94987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] avg=32768.0000 max=33097 max_rank=3 min=31672 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=33790 max_load_reduction_pct=0.505180 moved_to_replicas=70120 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 65369, 65380, 267, 32618, 32733, 347] avg=32768.0000 max=65380 max_rank=3 min=267 min_rank=4 imbalance_ratio=1.995239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[62793, 66887, 32629, 27043, 157, 39728, 32667, 240] avg=32768.0000 max=66887 max_rank=1 min=157 min_rank=4 imbalance_ratio=2.041229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[32065, 445, 32657, 32773, 66691, 64522, 32781, 210] avg=32768.0000 max=66691 max_rank=4 min=210 min_rank=7 imbalance_ratio=2.035248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] avg=32768.0000 max=33096 max_rank=2 min=32551 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32284 max_load_reduction_pct=0.493790 moved_to_replicas=64557 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 65369, 65380, 267, 32618, 32733, 347] avg=32768.0000 max=65380 max_rank=3 min=267 min_rank=4 imbalance_ratio=1.995239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[62793, 66887, 32629, 27043, 157, 39728, 32667, 240] avg=32768.0000 max=66887 max_rank=1 min=157 min_rank=4 imbalance_ratio=2.041229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[62793, 66887, 32629, 27043, 157, 39728, 32667, 240] avg=32768.0000 max=66887 max_rank=1 min=157 min_rank=4 imbalance_ratio=2.041229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[22147, 59841, 48209, 32667, 271, 32580, 33521, 32908] avg=32768.0000 max=59841 max_rank=1 min=271 min_rank=4 imbalance_ratio=1.826202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[3, 32771, 98259, 32909, 11, 32643, 32487, 33061] avg=32768.0000 max=98259 max_rank=2 min=3 min_rank=0 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] avg=32768.0000 max=33097 max_rank=3 min=31672 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=33790 max_load_reduction_pct=0.505180 moved_to_replicas=70120 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[32065, 32832, 32657, 32773, 33096, 33096, 32781, 32844] avg=32768.0000 max=33096 max_rank=4 min=32065 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33595 max_load_reduction_pct=0.503741 moved_to_replicas=65021 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] avg=32768.0000 max=33096 max_rank=2 min=32551 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32284 max_load_reduction_pct=0.493790 moved_to_replicas=64557 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] avg=32768.0000 max=33097 max_rank=3 min=31672 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=33790 max_load_reduction_pct=0.505180 moved_to_replicas=70120 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] avg=32768.0000 max=33097 max_rank=3 min=31672 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=33790 max_load_reduction_pct=0.505180 moved_to_replicas=70120 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] avg=32768.0000 max=33097 max_rank=1 min=31604 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=26744 max_load_reduction_pct=0.446918 moved_to_replicas=42283 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] avg=32768.0000 max=33096 max_rank=2 min=32487 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=65163 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[62793, 66887, 32629, 27043, 157, 39728, 32667, 240] avg=32768.0000 max=66887 max_rank=1 min=157 min_rank=4 imbalance_ratio=2.041229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[3, 32771, 98259, 32909, 11, 32643, 32487, 33061] avg=32768.0000 max=98259 max_rank=2 min=3 min_rank=0 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[32666, 757, 32891, 12279, 50350, 32832, 32479, 67890] avg=32768.0000 max=67890 max_rank=7 min=757 min_rank=1 imbalance_ratio=2.071838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[62793, 66887, 32629, 27043, 157, 39728, 32667, 240] avg=32768.0000 max=66887 max_rank=1 min=157 min_rank=4 imbalance_ratio=2.041229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] avg=32768.0000 max=33097 max_rank=3 min=31672 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=33790 max_load_reduction_pct=0.505180 moved_to_replicas=70120 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[3, 32771, 98259, 32909, 11, 32643, 32487, 33061] avg=32768.0000 max=98259 max_rank=2 min=3 min_rank=0 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[3, 32771, 98259, 32909, 11, 32643, 32487, 33061] avg=32768.0000 max=98259 max_rank=2 min=3 min_rank=0 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[32742, 65273, 65470, 13, 32829, 32648, 33026, 143] avg=32768.0000 max=65470 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 65369, 65380, 267, 32618, 32733, 347] avg=32768.0000 max=65380 max_rank=3 min=267 min_rank=4 imbalance_ratio=1.995239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] avg=32768.0000 max=33096 max_rank=2 min=32487 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=65163 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[32666, 33096, 32891, 31987, 33097, 32832, 32479, 33096] avg=32768.0000 max=33097 max_rank=4 min=31987 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=34793 max_load_reduction_pct=0.512491 moved_to_replicas=52048 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] avg=32768.0000 max=33097 max_rank=3 min=31672 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=33790 max_load_reduction_pct=0.505180 moved_to_replicas=70120 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] avg=32768.0000 max=33096 max_rank=2 min=32487 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=65163 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] avg=32768.0000 max=33096 max_rank=2 min=32487 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=65163 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32374 max_load_reduction_pct=0.494486 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[3, 32771, 98259, 32909, 11, 32643, 32487, 33061] avg=32768.0000 max=98259 max_rank=2 min=3 min_rank=0 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] avg=32768.0000 max=33096 max_rank=2 min=32551 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32284 max_load_reduction_pct=0.493790 moved_to_replicas=64557 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[32742, 65273, 65470, 13, 32829, 32648, 33026, 143] avg=32768.0000 max=65470 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[32905, 64000, 13894, 268, 1126, 64897, 19676, 65378] avg=32768.0000 max=65378 max_rank=7 min=268 min_rank=3 imbalance_ratio=1.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[3, 32771, 98259, 32909, 11, 32643, 32487, 33061] avg=32768.0000 max=98259 max_rank=2 min=3 min_rank=0 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] avg=32768.0000 max=33096 max_rank=2 min=32487 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=65163 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[32742, 65273, 65470, 13, 32829, 32648, 33026, 143] avg=32768.0000 max=65470 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[32742, 65273, 65470, 13, 32829, 32648, 33026, 143] avg=32768.0000 max=65470 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 96092, 27, 90, 67922, 158, 32289] avg=32768.0000 max=96092 max_rank=2 min=27 min_rank=3 imbalance_ratio=2.932495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32374 max_load_reduction_pct=0.494486 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[62793, 66887, 32629, 27043, 157, 39728, 32667, 240] avg=32768.0000 max=66887 max_rank=1 min=157 min_rank=4 imbalance_ratio=2.041229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[32905, 33096, 33096, 32550, 32927, 33096, 31378, 33096] avg=32768.0000 max=33096 max_rank=1 min=31378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32282 max_load_reduction_pct=0.493775 moved_to_replicas=94987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] avg=32768.0000 max=33096 max_rank=2 min=32487 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=65163 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32374 max_load_reduction_pct=0.494486 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32374 max_load_reduction_pct=0.494486 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] avg=32768.0000 max=33096 max_rank=2 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62996 max_load_reduction_pct=0.655580 moved_to_replicas=97822 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[32742, 65273, 65470, 13, 32829, 32648, 33026, 143] avg=32768.0000 max=65470 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] avg=32768.0000 max=33097 max_rank=3 min=31672 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=33790 max_load_reduction_pct=0.505180 moved_to_replicas=70120 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 96092, 27, 90, 67922, 158, 32289] avg=32768.0000 max=96092 max_rank=2 min=27 min_rank=3 imbalance_ratio=2.932495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[22147, 59841, 48209, 32667, 271, 32580, 33521, 32908] avg=32768.0000 max=59841 max_rank=1 min=271 min_rank=4 imbalance_ratio=1.826202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[32742, 65273, 65470, 13, 32829, 32648, 33026, 143] avg=32768.0000 max=65470 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32374 max_load_reduction_pct=0.494486 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 96092, 27, 90, 67922, 158, 32289] avg=32768.0000 max=96092 max_rank=2 min=27 min_rank=3 imbalance_ratio=2.932495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 96092, 27, 90, 67922, 158, 32289] avg=32768.0000 max=96092 max_rank=2 min=27 min_rank=3 imbalance_ratio=2.932495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=26 tokens_per_ep_rank=[28915, 132, 97933, 7, 97920, 5524, 31710, 3] avg=32768.0000 max=97933 max_rank=2 min=3 min_rank=7 imbalance_ratio=2.988678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] avg=32768.0000 max=33096 max_rank=2 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62996 max_load_reduction_pct=0.655580 moved_to_replicas=97822 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[3, 32771, 98259, 32909, 11, 32643, 32487, 33061] avg=32768.0000 max=98259 max_rank=2 min=3 min_rank=0 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[33095, 33097, 33097, 32667, 31604, 32580, 33096, 32908] avg=32768.0000 max=33097 max_rank=1 min=31604 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=26744 max_load_reduction_pct=0.446918 moved_to_replicas=42283 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32374 max_load_reduction_pct=0.494486 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] avg=32768.0000 max=33096 max_rank=2 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62996 max_load_reduction_pct=0.655580 moved_to_replicas=97822 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] avg=32768.0000 max=33096 max_rank=2 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62996 max_load_reduction_pct=0.655580 moved_to_replicas=97822 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=26 tokens_per_ep_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] avg=32768.0000 max=33097 max_rank=0 min=32143 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64836 max_load_reduction_pct=0.662044 moved_to_replicas=129661 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 96092, 27, 90, 67922, 158, 32289] avg=32768.0000 max=96092 max_rank=2 min=27 min_rank=3 imbalance_ratio=2.932495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] avg=32768.0000 max=33096 max_rank=2 min=32487 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=65163 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=26 tokens_per_ep_rank=[28915, 132, 97933, 7, 97920, 5524, 31710, 3] avg=32768.0000 max=97933 max_rank=2 min=3 min_rank=7 imbalance_ratio=2.988678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 65369, 65380, 267, 32618, 32733, 347] avg=32768.0000 max=65380 max_rank=3 min=267 min_rank=4 imbalance_ratio=1.995239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 96092, 27, 90, 67922, 158, 32289] avg=32768.0000 max=96092 max_rank=2 min=27 min_rank=3 imbalance_ratio=2.932495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=26 tokens_per_ep_rank=[28915, 132, 97933, 7, 97920, 5524, 31710, 3] avg=32768.0000 max=97933 max_rank=2 min=3 min_rank=7 imbalance_ratio=2.988678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] avg=32768.0000 max=33096 max_rank=2 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62996 max_load_reduction_pct=0.655580 moved_to_replicas=97822 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=26 tokens_per_ep_rank=[28915, 132, 97933, 7, 97920, 5524, 31710, 3] avg=32768.0000 max=97933 max_rank=2 min=3 min_rank=7 imbalance_ratio=2.988678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=27 tokens_per_ep_rank=[32935, 65318, 32642, 131, 65460, 443, 65166, 49] avg=32768.0000 max=65460 max_rank=4 min=49 min_rank=7 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=26 tokens_per_ep_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] avg=32768.0000 max=33097 max_rank=0 min=32143 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64836 max_load_reduction_pct=0.662044 moved_to_replicas=129661 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[32744, 32686, 33096, 33096, 32551, 32618, 32733, 32620] avg=32768.0000 max=33096 max_rank=2 min=32551 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32284 max_load_reduction_pct=0.493790 moved_to_replicas=64557 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[32742, 65273, 65470, 13, 32829, 32648, 33026, 143] avg=32768.0000 max=65470 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] avg=32768.0000 max=33096 max_rank=2 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62996 max_load_reduction_pct=0.655580 moved_to_replicas=97822 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=26 tokens_per_ep_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] avg=32768.0000 max=33097 max_rank=0 min=32143 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64836 max_load_reduction_pct=0.662044 moved_to_replicas=129661 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=26 tokens_per_ep_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] avg=32768.0000 max=33097 max_rank=0 min=32143 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64836 max_load_reduction_pct=0.662044 moved_to_replicas=129661 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=27 tokens_per_ep_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=96656 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=26 tokens_per_ep_rank=[28915, 132, 97933, 7, 97920, 5524, 31710, 3] avg=32768.0000 max=97933 max_rank=2 min=3 min_rank=7 imbalance_ratio=2.988678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32374 max_load_reduction_pct=0.494486 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=27 tokens_per_ep_rank=[32935, 65318, 32642, 131, 65460, 443, 65166, 49] avg=32768.0000 max=65460 max_rank=4 min=49 min_rank=7 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[62793, 66887, 32629, 27043, 157, 39728, 32667, 240] avg=32768.0000 max=66887 max_rank=1 min=157 min_rank=4 imbalance_ratio=2.041229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=26 tokens_per_ep_rank=[28915, 132, 97933, 7, 97920, 5524, 31710, 3] avg=32768.0000 max=97933 max_rank=2 min=3 min_rank=7 imbalance_ratio=2.988678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=26 tokens_per_ep_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] avg=32768.0000 max=33097 max_rank=0 min=32143 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64836 max_load_reduction_pct=0.662044 moved_to_replicas=129661 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=28 tokens_per_ep_rank=[48, 65242, 32905, 212, 502, 65333, 32615, 65287] avg=32768.0000 max=65333 max_rank=5 min=48 min_rank=0 imbalance_ratio=1.993805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=27 tokens_per_ep_rank=[32935, 65318, 32642, 131, 65460, 443, 65166, 49] avg=32768.0000 max=65460 max_rank=4 min=49 min_rank=7 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=27 tokens_per_ep_rank=[32935, 65318, 32642, 131, 65460, 443, 65166, 49] avg=32768.0000 max=65460 max_rank=4 min=49 min_rank=7 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=27 tokens_per_ep_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=96656 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 96092, 27, 90, 67922, 158, 32289] avg=32768.0000 max=96092 max_rank=2 min=27 min_rank=3 imbalance_ratio=2.932495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=26 tokens_per_ep_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] avg=32768.0000 max=33097 max_rank=0 min=32143 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64836 max_load_reduction_pct=0.662044 moved_to_replicas=129661 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[33096, 33096, 32629, 33097, 32792, 33095, 32667, 31672] avg=32768.0000 max=33097 max_rank=3 min=31672 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=33790 max_load_reduction_pct=0.505180 moved_to_replicas=70120 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=28 tokens_per_ep_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] avg=32768.0000 max=33096 max_rank=1 min=32285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32237 max_load_reduction_pct=0.493426 moved_to_replicas=96574 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=27 tokens_per_ep_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=96656 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=27 tokens_per_ep_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=96656 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=27 tokens_per_ep_rank=[32935, 65318, 32642, 131, 65460, 443, 65166, 49] avg=32768.0000 max=65460 max_rank=4 min=49 min_rank=7 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=28 tokens_per_ep_rank=[48, 65242, 32905, 212, 502, 65333, 32615, 65287] avg=32768.0000 max=65333 max_rank=5 min=48 min_rank=0 imbalance_ratio=1.993805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] avg=32768.0000 max=33096 max_rank=2 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62996 max_load_reduction_pct=0.655580 moved_to_replicas=97822 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=27 tokens_per_ep_rank=[32935, 65318, 32642, 131, 65460, 443, 65166, 49] avg=32768.0000 max=65460 max_rank=4 min=49 min_rank=7 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=27 tokens_per_ep_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=96656 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[3, 32771, 98259, 32909, 11, 32643, 32487, 33061] avg=32768.0000 max=98259 max_rank=2 min=3 min_rank=0 imbalance_ratio=2.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=29 tokens_per_ep_rank=[302, 47, 129539, 265, 65401, 65852, 96, 642] avg=32768.0000 max=129539 max_rank=2 min=47 min_rank=1 imbalance_ratio=3.953217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=28 tokens_per_ep_rank=[48, 65242, 32905, 212, 502, 65333, 32615, 65287] avg=32768.0000 max=65333 max_rank=5 min=48 min_rank=0 imbalance_ratio=1.993805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=28 tokens_per_ep_rank=[48, 65242, 32905, 212, 502, 65333, 32615, 65287] avg=32768.0000 max=65333 max_rank=5 min=48 min_rank=0 imbalance_ratio=1.993805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=28 tokens_per_ep_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] avg=32768.0000 max=33096 max_rank=1 min=32285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32237 max_load_reduction_pct=0.493426 moved_to_replicas=96574 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=26 tokens_per_ep_rank=[28915, 132, 97933, 7, 97920, 5524, 31710, 3] avg=32768.0000 max=97933 max_rank=2 min=3 min_rank=7 imbalance_ratio=2.988678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=27 tokens_per_ep_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=96656 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[32639, 32771, 33096, 32909, 32538, 32643, 32487, 33061] avg=32768.0000 max=33096 max_rank=2 min=32487 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=65163 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=29 tokens_per_ep_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] avg=32768.0000 max=33096 max_rank=2 min=31441 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96443 max_load_reduction_pct=0.744509 moved_to_replicas=161504 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=28 tokens_per_ep_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] avg=32768.0000 max=33096 max_rank=1 min=32285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32237 max_load_reduction_pct=0.493426 moved_to_replicas=96574 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=28 tokens_per_ep_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] avg=32768.0000 max=33096 max_rank=1 min=32285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32237 max_load_reduction_pct=0.493426 moved_to_replicas=96574 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=28 tokens_per_ep_rank=[48, 65242, 32905, 212, 502, 65333, 32615, 65287] avg=32768.0000 max=65333 max_rank=5 min=48 min_rank=0 imbalance_ratio=1.993805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=26 tokens_per_ep_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] avg=32768.0000 max=33097 max_rank=0 min=32143 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64836 max_load_reduction_pct=0.662044 moved_to_replicas=129661 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=29 tokens_per_ep_rank=[302, 47, 129539, 265, 65401, 65852, 96, 642] avg=32768.0000 max=129539 max_rank=2 min=47 min_rank=1 imbalance_ratio=3.953217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=28 tokens_per_ep_rank=[48, 65242, 32905, 212, 502, 65333, 32615, 65287] avg=32768.0000 max=65333 max_rank=5 min=48 min_rank=0 imbalance_ratio=1.993805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=28 tokens_per_ep_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] avg=32768.0000 max=33096 max_rank=1 min=32285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32237 max_load_reduction_pct=0.493426 moved_to_replicas=96574 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[32742, 65273, 65470, 13, 32829, 32648, 33026, 143] avg=32768.0000 max=65470 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=30 tokens_per_ep_rank=[32780, 97853, 32767, 32801, 32804, 139, 126, 32874] avg=32768.0000 max=97853 max_rank=1 min=126 min_rank=6 imbalance_ratio=2.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=29 tokens_per_ep_rank=[302, 47, 129539, 265, 65401, 65852, 96, 642] avg=32768.0000 max=129539 max_rank=2 min=47 min_rank=1 imbalance_ratio=3.953217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=29 tokens_per_ep_rank=[302, 47, 129539, 265, 65401, 65852, 96, 642] avg=32768.0000 max=129539 max_rank=2 min=47 min_rank=1 imbalance_ratio=3.953217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=29 tokens_per_ep_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] avg=32768.0000 max=33096 max_rank=2 min=31441 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96443 max_load_reduction_pct=0.744509 moved_to_replicas=161504 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=27 tokens_per_ep_rank=[32935, 65318, 32642, 131, 65460, 443, 65166, 49] avg=32768.0000 max=65460 max_rank=4 min=49 min_rank=7 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=28 tokens_per_ep_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] avg=32768.0000 max=33096 max_rank=1 min=32285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32237 max_load_reduction_pct=0.493426 moved_to_replicas=96574 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[32742, 33096, 33096, 32387, 32829, 32648, 33026, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32374 max_load_reduction_pct=0.494486 moved_to_replicas=64551 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=30 tokens_per_ep_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] avg=32768.0000 max=33096 max_rank=1 min=32263 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64757 max_load_reduction_pct=0.661778 moved_to_replicas=64757 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=29 tokens_per_ep_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] avg=32768.0000 max=33096 max_rank=2 min=31441 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96443 max_load_reduction_pct=0.744509 moved_to_replicas=161504 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=29 tokens_per_ep_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] avg=32768.0000 max=33096 max_rank=2 min=31441 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96443 max_load_reduction_pct=0.744509 moved_to_replicas=161504 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=29 tokens_per_ep_rank=[302, 47, 129539, 265, 65401, 65852, 96, 642] avg=32768.0000 max=129539 max_rank=2 min=47 min_rank=1 imbalance_ratio=3.953217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=27 tokens_per_ep_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=96656 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=30 tokens_per_ep_rank=[32780, 97853, 32767, 32801, 32804, 139, 126, 32874] avg=32768.0000 max=97853 max_rank=1 min=126 min_rank=6 imbalance_ratio=2.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=29 tokens_per_ep_rank=[302, 47, 129539, 265, 65401, 65852, 96, 642] avg=32768.0000 max=129539 max_rank=2 min=47 min_rank=1 imbalance_ratio=3.953217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=29 tokens_per_ep_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] avg=32768.0000 max=33096 max_rank=2 min=31441 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96443 max_load_reduction_pct=0.744509 moved_to_replicas=161504 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 96092, 27, 90, 67922, 158, 32289] avg=32768.0000 max=96092 max_rank=2 min=27 min_rank=3 imbalance_ratio=2.932495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=31 tokens_per_ep_rank=[98088, 143, 30, 33119, 32683, 32563, 32753, 32765] avg=32768.0000 max=98088 max_rank=0 min=30 min_rank=2 imbalance_ratio=2.993408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=30 tokens_per_ep_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] avg=32768.0000 max=33096 max_rank=1 min=32263 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64757 max_load_reduction_pct=0.661778 moved_to_replicas=64757 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=30 tokens_per_ep_rank=[32780, 97853, 32767, 32801, 32804, 139, 126, 32874] avg=32768.0000 max=97853 max_rank=1 min=126 min_rank=6 imbalance_ratio=2.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=30 tokens_per_ep_rank=[32780, 97853, 32767, 32801, 32804, 139, 126, 32874] avg=32768.0000 max=97853 max_rank=1 min=126 min_rank=6 imbalance_ratio=2.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=29 tokens_per_ep_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] avg=32768.0000 max=33096 max_rank=2 min=31441 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96443 max_load_reduction_pct=0.744509 moved_to_replicas=161504 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=28 tokens_per_ep_rank=[48, 65242, 32905, 212, 502, 65333, 32615, 65287] avg=32768.0000 max=65333 max_rank=5 min=48 min_rank=0 imbalance_ratio=1.993805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=31 tokens_per_ep_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] avg=32768.0000 max=33096 max_rank=0 min=32525 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64992 max_load_reduction_pct=0.662589 moved_to_replicas=65015 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=25 tokens_per_ep_rank=[32723, 32843, 33096, 32665, 32638, 33096, 32794, 32289] avg=32768.0000 max=33096 max_rank=2 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62996 max_load_reduction_pct=0.655580 moved_to_replicas=97822 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=30 tokens_per_ep_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] avg=32768.0000 max=33096 max_rank=1 min=32263 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64757 max_load_reduction_pct=0.661778 moved_to_replicas=64757 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=30 tokens_per_ep_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] avg=32768.0000 max=33096 max_rank=1 min=32263 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64757 max_load_reduction_pct=0.661778 moved_to_replicas=64757 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=30 tokens_per_ep_rank=[32780, 97853, 32767, 32801, 32804, 139, 126, 32874] avg=32768.0000 max=97853 max_rank=1 min=126 min_rank=6 imbalance_ratio=2.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=31 tokens_per_ep_rank=[98088, 143, 30, 33119, 32683, 32563, 32753, 32765] avg=32768.0000 max=98088 max_rank=0 min=30 min_rank=2 imbalance_ratio=2.993408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=28 tokens_per_ep_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] avg=32768.0000 max=33096 max_rank=1 min=32285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32237 max_load_reduction_pct=0.493426 moved_to_replicas=96574 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=30 tokens_per_ep_rank=[32780, 97853, 32767, 32801, 32804, 139, 126, 32874] avg=32768.0000 max=97853 max_rank=1 min=126 min_rank=6 imbalance_ratio=2.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=30 tokens_per_ep_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] avg=32768.0000 max=33096 max_rank=1 min=32263 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64757 max_load_reduction_pct=0.661778 moved_to_replicas=64757 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=32 tokens_per_ep_rank=[65559, 32865, 65368, 65273, 154, 116, 159, 32650] avg=32768.0000 max=65559 max_rank=0 min=116 min_rank=5 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=26 tokens_per_ep_rank=[28915, 132, 97933, 7, 97920, 5524, 31710, 3] avg=32768.0000 max=97933 max_rank=2 min=3 min_rank=7 imbalance_ratio=2.988678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=31 tokens_per_ep_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] avg=32768.0000 max=33096 max_rank=0 min=32525 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64992 max_load_reduction_pct=0.662589 moved_to_replicas=65015 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=31 tokens_per_ep_rank=[98088, 143, 30, 33119, 32683, 32563, 32753, 32765] avg=32768.0000 max=98088 max_rank=0 min=30 min_rank=2 imbalance_ratio=2.993408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=31 tokens_per_ep_rank=[98088, 143, 30, 33119, 32683, 32563, 32753, 32765] avg=32768.0000 max=98088 max_rank=0 min=30 min_rank=2 imbalance_ratio=2.993408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=30 tokens_per_ep_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] avg=32768.0000 max=33096 max_rank=1 min=32263 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64757 max_load_reduction_pct=0.661778 moved_to_replicas=64757 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=29 tokens_per_ep_rank=[302, 47, 129539, 265, 65401, 65852, 96, 642] avg=32768.0000 max=129539 max_rank=2 min=47 min_rank=1 imbalance_ratio=3.953217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] avg=32768.0000 max=33096 max_rank=0 min=32336 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=96912 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=26 tokens_per_ep_rank=[33097, 32770, 33096, 32206, 33096, 33095, 32143, 32641] avg=32768.0000 max=33097 max_rank=0 min=32143 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=64836 max_load_reduction_pct=0.662044 moved_to_replicas=129661 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=31 tokens_per_ep_rank=[98088, 143, 30, 33119, 32683, 32563, 32753, 32765] avg=32768.0000 max=98088 max_rank=0 min=30 min_rank=2 imbalance_ratio=2.993408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=31 tokens_per_ep_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] avg=32768.0000 max=33096 max_rank=0 min=32525 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64992 max_load_reduction_pct=0.662589 moved_to_replicas=65015 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=31 tokens_per_ep_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] avg=32768.0000 max=33096 max_rank=0 min=32525 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64992 max_load_reduction_pct=0.662589 moved_to_replicas=65015 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=32 tokens_per_ep_rank=[65559, 32865, 65368, 65273, 154, 116, 159, 32650] avg=32768.0000 max=65559 max_rank=0 min=116 min_rank=5 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=29 tokens_per_ep_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] avg=32768.0000 max=33096 max_rank=2 min=31441 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96443 max_load_reduction_pct=0.744509 moved_to_replicas=161504 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=31 tokens_per_ep_rank=[98088, 143, 30, 33119, 32683, 32563, 32753, 32765] avg=32768.0000 max=98088 max_rank=0 min=30 min_rank=2 imbalance_ratio=2.993408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=31 tokens_per_ep_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] avg=32768.0000 max=33096 max_rank=0 min=32525 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64992 max_load_reduction_pct=0.662589 moved_to_replicas=65015 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=33 tokens_per_ep_rank=[65118, 65438, 32640, 32992, 132, 65291, 404, 129] avg=32768.0000 max=65438 max_rank=1 min=129 min_rank=7 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=27 tokens_per_ep_rank=[32935, 65318, 32642, 131, 65460, 443, 65166, 49] avg=32768.0000 max=65460 max_rank=4 min=49 min_rank=7 imbalance_ratio=1.997681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] avg=32768.0000 max=33096 max_rank=0 min=32336 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=96912 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=32 tokens_per_ep_rank=[65559, 32865, 65368, 65273, 154, 116, 159, 32650] avg=32768.0000 max=65559 max_rank=0 min=116 min_rank=5 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=32 tokens_per_ep_rank=[65559, 32865, 65368, 65273, 154, 116, 159, 32650] avg=32768.0000 max=65559 max_rank=0 min=116 min_rank=5 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=31 tokens_per_ep_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] avg=32768.0000 max=33096 max_rank=0 min=32525 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64992 max_load_reduction_pct=0.662589 moved_to_replicas=65015 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=30 tokens_per_ep_rank=[32780, 97853, 32767, 32801, 32804, 139, 126, 32874] avg=32768.0000 max=97853 max_rank=1 min=126 min_rank=6 imbalance_ratio=2.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] avg=32768.0000 max=33096 max_rank=0 min=32327 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=96559 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=27 tokens_per_ep_rank=[32935, 33096, 32642, 32353, 33096, 32513, 33096, 32413] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32364 max_load_reduction_pct=0.494409 moved_to_replicas=96656 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] avg=32768.0000 max=33096 max_rank=0 min=32336 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=96912 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=32 tokens_per_ep_rank=[65559, 32865, 65368, 65273, 154, 116, 159, 32650] avg=32768.0000 max=65559 max_rank=0 min=116 min_rank=5 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] avg=32768.0000 max=33096 max_rank=0 min=32336 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=96912 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=33 tokens_per_ep_rank=[65118, 65438, 32640, 32992, 132, 65291, 404, 129] avg=32768.0000 max=65438 max_rank=1 min=129 min_rank=7 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=30 tokens_per_ep_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] avg=32768.0000 max=33096 max_rank=1 min=32263 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64757 max_load_reduction_pct=0.661778 moved_to_replicas=64757 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=32 tokens_per_ep_rank=[65559, 32865, 65368, 65273, 154, 116, 159, 32650] avg=32768.0000 max=65559 max_rank=0 min=116 min_rank=5 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=34 tokens_per_ep_rank=[65286, 226, 79, 197, 96711, 3, 32645, 66997] avg=32768.0000 max=96711 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.951385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] avg=32768.0000 max=33096 max_rank=0 min=32336 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=96912 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=28 tokens_per_ep_rank=[48, 65242, 32905, 212, 502, 65333, 32615, 65287] avg=32768.0000 max=65333 max_rank=5 min=48 min_rank=0 imbalance_ratio=1.993805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] avg=32768.0000 max=33096 max_rank=0 min=32327 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=96559 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=33 tokens_per_ep_rank=[65118, 65438, 32640, 32992, 132, 65291, 404, 129] avg=32768.0000 max=65438 max_rank=1 min=129 min_rank=7 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] avg=32768.0000 max=33096 max_rank=0 min=32336 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=96912 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=33 tokens_per_ep_rank=[65118, 65438, 32640, 32992, 132, 65291, 404, 129] avg=32768.0000 max=65438 max_rank=1 min=129 min_rank=7 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=34 tokens_per_ep_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] avg=32768.0000 max=33096 max_rank=0 min=31649 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63615 max_load_reduction_pct=0.657785 moved_to_replicas=129706 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=31 tokens_per_ep_rank=[98088, 143, 30, 33119, 32683, 32563, 32753, 32765] avg=32768.0000 max=98088 max_rank=0 min=30 min_rank=2 imbalance_ratio=2.993408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=28 tokens_per_ep_rank=[32285, 33096, 32905, 32403, 32648, 33096, 32615, 33096] avg=32768.0000 max=33096 max_rank=1 min=32285 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32237 max_load_reduction_pct=0.493426 moved_to_replicas=96574 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] avg=32768.0000 max=33096 max_rank=0 min=32327 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=96559 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=33 tokens_per_ep_rank=[65118, 65438, 32640, 32992, 132, 65291, 404, 129] avg=32768.0000 max=65438 max_rank=1 min=129 min_rank=7 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] avg=32768.0000 max=33096 max_rank=0 min=32327 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=96559 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=34 tokens_per_ep_rank=[65286, 226, 79, 197, 96711, 3, 32645, 66997] avg=32768.0000 max=96711 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.951385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=31 tokens_per_ep_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] avg=32768.0000 max=33096 max_rank=0 min=32525 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64992 max_load_reduction_pct=0.662589 moved_to_replicas=65015 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=33 tokens_per_ep_rank=[65118, 65438, 32640, 32992, 132, 65291, 404, 129] avg=32768.0000 max=65438 max_rank=1 min=129 min_rank=7 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=35 tokens_per_ep_rank=[32900, 138, 33038, 32653, 32565, 26648, 38936, 65266] avg=32768.0000 max=65266 max_rank=7 min=138 min_rank=1 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] avg=32768.0000 max=33096 max_rank=0 min=32327 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=96559 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=34 tokens_per_ep_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] avg=32768.0000 max=33096 max_rank=0 min=31649 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63615 max_load_reduction_pct=0.657785 moved_to_replicas=129706 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=29 tokens_per_ep_rank=[302, 47, 129539, 265, 65401, 65852, 96, 642] avg=32768.0000 max=129539 max_rank=2 min=47 min_rank=1 imbalance_ratio=3.953217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=34 tokens_per_ep_rank=[65286, 226, 79, 197, 96711, 3, 32645, 66997] avg=32768.0000 max=96711 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.951385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] avg=32768.0000 max=33096 max_rank=0 min=32327 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=96559 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=34 tokens_per_ep_rank=[65286, 226, 79, 197, 96711, 3, 32645, 66997] avg=32768.0000 max=96711 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.951385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=35 tokens_per_ep_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=32308 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32168 max_load_reduction_pct=0.492875 moved_to_replicas=38010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=32 tokens_per_ep_rank=[65559, 32865, 65368, 65273, 154, 116, 159, 32650] avg=32768.0000 max=65559 max_rank=0 min=116 min_rank=5 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=29 tokens_per_ep_rank=[32938, 32682, 33096, 31441, 33096, 33096, 32728, 33067] avg=32768.0000 max=33096 max_rank=2 min=31441 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96443 max_load_reduction_pct=0.744509 moved_to_replicas=161504 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=34 tokens_per_ep_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] avg=32768.0000 max=33096 max_rank=0 min=31649 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63615 max_load_reduction_pct=0.657785 moved_to_replicas=129706 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=34 tokens_per_ep_rank=[65286, 226, 79, 197, 96711, 3, 32645, 66997] avg=32768.0000 max=96711 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.951385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=35 tokens_per_ep_rank=[32900, 138, 33038, 32653, 32565, 26648, 38936, 65266] avg=32768.0000 max=65266 max_rank=7 min=138 min_rank=1 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=34 tokens_per_ep_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] avg=32768.0000 max=33096 max_rank=0 min=31649 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63615 max_load_reduction_pct=0.657785 moved_to_replicas=129706 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] avg=32768.0000 max=33096 max_rank=0 min=32336 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=96912 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=34 tokens_per_ep_rank=[65286, 226, 79, 197, 96711, 3, 32645, 66997] avg=32768.0000 max=96711 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.951385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=36 tokens_per_ep_rank=[185, 163216, 152, 32664, 136, 32875, 276, 32640] avg=32768.0000 max=163216 max_rank=1 min=136 min_rank=4 imbalance_ratio=4.980957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=34 tokens_per_ep_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] avg=32768.0000 max=33096 max_rank=0 min=31649 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63615 max_load_reduction_pct=0.657785 moved_to_replicas=129706 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=35 tokens_per_ep_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=32308 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32168 max_load_reduction_pct=0.492875 moved_to_replicas=38010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=35 tokens_per_ep_rank=[32900, 138, 33038, 32653, 32565, 26648, 38936, 65266] avg=32768.0000 max=65266 max_rank=7 min=138 min_rank=1 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=30 tokens_per_ep_rank=[32780, 97853, 32767, 32801, 32804, 139, 126, 32874] avg=32768.0000 max=97853 max_rank=1 min=126 min_rank=6 imbalance_ratio=2.986237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=34 tokens_per_ep_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] avg=32768.0000 max=33096 max_rank=0 min=31649 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63615 max_load_reduction_pct=0.657785 moved_to_replicas=129706 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=35 tokens_per_ep_rank=[32900, 138, 33038, 32653, 32565, 26648, 38936, 65266] avg=32768.0000 max=65266 max_rank=7 min=138 min_rank=1 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=36 tokens_per_ep_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] avg=32768.0000 max=33096 max_rank=1 min=32510 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130120 max_load_reduction_pct=0.797226 moved_to_replicas=130120 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=33 tokens_per_ep_rank=[65118, 65438, 32640, 32992, 132, 65291, 404, 129] avg=32768.0000 max=65438 max_rank=1 min=129 min_rank=7 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=35 tokens_per_ep_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=32308 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32168 max_load_reduction_pct=0.492875 moved_to_replicas=38010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=30 tokens_per_ep_rank=[32780, 33096, 32767, 32801, 32804, 32263, 32759, 32874] avg=32768.0000 max=33096 max_rank=1 min=32263 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64757 max_load_reduction_pct=0.661778 moved_to_replicas=64757 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=35 tokens_per_ep_rank=[32900, 138, 33038, 32653, 32565, 26648, 38936, 65266] avg=32768.0000 max=65266 max_rank=7 min=138 min_rank=1 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=36 tokens_per_ep_rank=[185, 163216, 152, 32664, 136, 32875, 276, 32640] avg=32768.0000 max=163216 max_rank=1 min=136 min_rank=4 imbalance_ratio=4.980957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=35 tokens_per_ep_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=32308 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32168 max_load_reduction_pct=0.492875 moved_to_replicas=38010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] avg=32768.0000 max=33096 max_rank=0 min=32327 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=96559 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=35 tokens_per_ep_rank=[32900, 138, 33038, 32653, 32565, 26648, 38936, 65266] avg=32768.0000 max=65266 max_rank=7 min=138 min_rank=1 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=37 tokens_per_ep_rank=[32639, 3194, 268, 95217, 32899, 32467, 57, 65403] avg=32768.0000 max=95217 max_rank=3 min=57 min_rank=6 imbalance_ratio=2.905792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=35 tokens_per_ep_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=32308 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32168 max_load_reduction_pct=0.492875 moved_to_replicas=38010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=36 tokens_per_ep_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] avg=32768.0000 max=33096 max_rank=1 min=32510 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130120 max_load_reduction_pct=0.797226 moved_to_replicas=130120 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=36 tokens_per_ep_rank=[185, 163216, 152, 32664, 136, 32875, 276, 32640] avg=32768.0000 max=163216 max_rank=1 min=136 min_rank=4 imbalance_ratio=4.980957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=31 tokens_per_ep_rank=[98088, 143, 30, 33119, 32683, 32563, 32753, 32765] avg=32768.0000 max=98088 max_rank=0 min=30 min_rank=2 imbalance_ratio=2.993408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=36 tokens_per_ep_rank=[185, 163216, 152, 32664, 136, 32875, 276, 32640] avg=32768.0000 max=163216 max_rank=1 min=136 min_rank=4 imbalance_ratio=4.980957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=35 tokens_per_ep_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=32308 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32168 max_load_reduction_pct=0.492875 moved_to_replicas=38010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=37 tokens_per_ep_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] avg=32768.0000 max=33096 max_rank=1 min=32162 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62121 max_load_reduction_pct=0.652415 moved_to_replicas=94428 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=36 tokens_per_ep_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] avg=32768.0000 max=33096 max_rank=1 min=32510 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130120 max_load_reduction_pct=0.797226 moved_to_replicas=130120 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=31 tokens_per_ep_rank=[33096, 32525, 32663, 33096, 32683, 32563, 32753, 32765] avg=32768.0000 max=33096 max_rank=0 min=32525 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64992 max_load_reduction_pct=0.662589 moved_to_replicas=65015 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=34 tokens_per_ep_rank=[65286, 226, 79, 197, 96711, 3, 32645, 66997] avg=32768.0000 max=96711 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.951385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=36 tokens_per_ep_rank=[185, 163216, 152, 32664, 136, 32875, 276, 32640] avg=32768.0000 max=163216 max_rank=1 min=136 min_rank=4 imbalance_ratio=4.980957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=37 tokens_per_ep_rank=[32639, 3194, 268, 95217, 32899, 32467, 57, 65403] avg=32768.0000 max=95217 max_rank=3 min=57 min_rank=6 imbalance_ratio=2.905792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=36 tokens_per_ep_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] avg=32768.0000 max=33096 max_rank=1 min=32510 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130120 max_load_reduction_pct=0.797226 moved_to_replicas=130120 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=36 tokens_per_ep_rank=[185, 163216, 152, 32664, 136, 32875, 276, 32640] avg=32768.0000 max=163216 max_rank=1 min=136 min_rank=4 imbalance_ratio=4.980957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=38 tokens_per_ep_rank=[32755, 65429, 32675, 65582, 259, 2, 252, 65190] avg=32768.0000 max=65582 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.001404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=34 tokens_per_ep_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] avg=32768.0000 max=33096 max_rank=0 min=31649 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63615 max_load_reduction_pct=0.657785 moved_to_replicas=129706 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=36 tokens_per_ep_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] avg=32768.0000 max=33096 max_rank=1 min=32510 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130120 max_load_reduction_pct=0.797226 moved_to_replicas=130120 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=37 tokens_per_ep_rank=[32639, 3194, 268, 95217, 32899, 32467, 57, 65403] avg=32768.0000 max=95217 max_rank=3 min=57 min_rank=6 imbalance_ratio=2.905792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=37 tokens_per_ep_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] avg=32768.0000 max=33096 max_rank=1 min=32162 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62121 max_load_reduction_pct=0.652415 moved_to_replicas=94428 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=32 tokens_per_ep_rank=[65559, 32865, 65368, 65273, 154, 116, 159, 32650] avg=32768.0000 max=65559 max_rank=0 min=116 min_rank=5 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=36 tokens_per_ep_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] avg=32768.0000 max=33096 max_rank=1 min=32510 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130120 max_load_reduction_pct=0.797226 moved_to_replicas=130120 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=37 tokens_per_ep_rank=[32639, 3194, 268, 95217, 32899, 32467, 57, 65403] avg=32768.0000 max=95217 max_rank=3 min=57 min_rank=6 imbalance_ratio=2.905792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=38 tokens_per_ep_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32486 max_load_reduction_pct=0.495349 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=37 tokens_per_ep_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] avg=32768.0000 max=33096 max_rank=1 min=32162 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62121 max_load_reduction_pct=0.652415 moved_to_replicas=94428 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=32 tokens_per_ep_rank=[33096, 32865, 33096, 33096, 32426, 32579, 32336, 32650] avg=32768.0000 max=33096 max_rank=0 min=32336 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=96912 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=35 tokens_per_ep_rank=[32900, 138, 33038, 32653, 32565, 26648, 38936, 65266] avg=32768.0000 max=65266 max_rank=7 min=138 min_rank=1 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=37 tokens_per_ep_rank=[32639, 3194, 268, 95217, 32899, 32467, 57, 65403] avg=32768.0000 max=95217 max_rank=3 min=57 min_rank=6 imbalance_ratio=2.905792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=38 tokens_per_ep_rank=[32755, 65429, 32675, 65582, 259, 2, 252, 65190] avg=32768.0000 max=65582 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.001404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=37 tokens_per_ep_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] avg=32768.0000 max=33096 max_rank=1 min=32162 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62121 max_load_reduction_pct=0.652415 moved_to_replicas=94428 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=37 tokens_per_ep_rank=[32639, 3194, 268, 95217, 32899, 32467, 57, 65403] avg=32768.0000 max=95217 max_rank=3 min=57 min_rank=6 imbalance_ratio=2.905792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=39 tokens_per_ep_rank=[32655, 97899, 32714, 388, 254, 65464, 32762, 8] avg=32768.0000 max=97899 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.987640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=38 tokens_per_ep_rank=[32755, 65429, 32675, 65582, 259, 2, 252, 65190] avg=32768.0000 max=65582 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.001404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=37 tokens_per_ep_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] avg=32768.0000 max=33096 max_rank=1 min=32162 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62121 max_load_reduction_pct=0.652415 moved_to_replicas=94428 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=35 tokens_per_ep_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=32308 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32168 max_load_reduction_pct=0.492875 moved_to_replicas=38010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=38 tokens_per_ep_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32486 max_load_reduction_pct=0.495349 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=33 tokens_per_ep_rank=[65118, 65438, 32640, 32992, 132, 65291, 404, 129] avg=32768.0000 max=65438 max_rank=1 min=129 min_rank=7 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=37 tokens_per_ep_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] avg=32768.0000 max=33096 max_rank=1 min=32162 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62121 max_load_reduction_pct=0.652415 moved_to_replicas=94428 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=38 tokens_per_ep_rank=[32755, 65429, 32675, 65582, 259, 2, 252, 65190] avg=32768.0000 max=65582 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.001404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=39 tokens_per_ep_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64803 max_load_reduction_pct=0.661937 moved_to_replicas=97171 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=38 tokens_per_ep_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32486 max_load_reduction_pct=0.495349 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=33 tokens_per_ep_rank=[33096, 33096, 32640, 32992, 32327, 33096, 32426, 32471] avg=32768.0000 max=33096 max_rank=0 min=32327 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=96559 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=38 tokens_per_ep_rank=[32755, 65429, 32675, 65582, 259, 2, 252, 65190] avg=32768.0000 max=65582 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.001404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=38 tokens_per_ep_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32486 max_load_reduction_pct=0.495349 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=36 tokens_per_ep_rank=[185, 163216, 152, 32664, 136, 32875, 276, 32640] avg=32768.0000 max=163216 max_rank=1 min=136 min_rank=4 imbalance_ratio=4.980957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=39 tokens_per_ep_rank=[32655, 97899, 32714, 388, 254, 65464, 32762, 8] avg=32768.0000 max=97899 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.987640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=38 tokens_per_ep_rank=[32755, 65429, 32675, 65582, 259, 2, 252, 65190] avg=32768.0000 max=65582 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.001404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=39 tokens_per_ep_rank=[32655, 97899, 32714, 388, 254, 65464, 32762, 8] avg=32768.0000 max=97899 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.987640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=40 tokens_per_ep_rank=[202, 914, 130506, 65493, 199, 31871, 32838, 121] avg=32768.0000 max=130506 max_rank=2 min=121 min_rank=7 imbalance_ratio=3.982727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=38 tokens_per_ep_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32486 max_load_reduction_pct=0.495349 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=36 tokens_per_ep_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] avg=32768.0000 max=33096 max_rank=1 min=32510 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130120 max_load_reduction_pct=0.797226 moved_to_replicas=130120 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=39 tokens_per_ep_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64803 max_load_reduction_pct=0.661937 moved_to_replicas=97171 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=38 tokens_per_ep_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32486 max_load_reduction_pct=0.495349 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=34 tokens_per_ep_rank=[65286, 226, 79, 197, 96711, 3, 32645, 66997] avg=32768.0000 max=96711 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.951385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=39 tokens_per_ep_rank=[32655, 97899, 32714, 388, 254, 65464, 32762, 8] avg=32768.0000 max=97899 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.987640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=39 tokens_per_ep_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64803 max_load_reduction_pct=0.661937 moved_to_replicas=97171 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=40 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] avg=32768.0000 max=33096 max_rank=1 min=32086 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97410 max_load_reduction_pct=0.746402 moved_to_replicas=129807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=39 tokens_per_ep_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64803 max_load_reduction_pct=0.661937 moved_to_replicas=97171 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=34 tokens_per_ep_rank=[33096, 33096, 31649, 32826, 33096, 32640, 32645, 33096] avg=32768.0000 max=33096 max_rank=0 min=31649 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63615 max_load_reduction_pct=0.657785 moved_to_replicas=129706 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=39 tokens_per_ep_rank=[32655, 97899, 32714, 388, 254, 65464, 32762, 8] avg=32768.0000 max=97899 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.987640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=37 tokens_per_ep_rank=[32639, 3194, 268, 95217, 32899, 32467, 57, 65403] avg=32768.0000 max=95217 max_rank=3 min=57 min_rank=6 imbalance_ratio=2.905792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=40 tokens_per_ep_rank=[202, 914, 130506, 65493, 199, 31871, 32838, 121] avg=32768.0000 max=130506 max_rank=2 min=121 min_rank=7 imbalance_ratio=3.982727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=39 tokens_per_ep_rank=[32655, 97899, 32714, 388, 254, 65464, 32762, 8] avg=32768.0000 max=97899 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.987640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=40 tokens_per_ep_rank=[202, 914, 130506, 65493, 199, 31871, 32838, 121] avg=32768.0000 max=130506 max_rank=2 min=121 min_rank=7 imbalance_ratio=3.982727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 97998, 65272, 32749, 19, 430, 185] avg=32768.0000 max=97998 max_rank=2 min=19 min_rank=5 imbalance_ratio=2.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=39 tokens_per_ep_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64803 max_load_reduction_pct=0.661937 moved_to_replicas=97171 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=40 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] avg=32768.0000 max=33096 max_rank=1 min=32086 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97410 max_load_reduction_pct=0.746402 moved_to_replicas=129807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=35 tokens_per_ep_rank=[32900, 138, 33038, 32653, 32565, 26648, 38936, 65266] avg=32768.0000 max=65266 max_rank=7 min=138 min_rank=1 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=37 tokens_per_ep_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] avg=32768.0000 max=33096 max_rank=1 min=32162 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62121 max_load_reduction_pct=0.652415 moved_to_replicas=94428 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=40 tokens_per_ep_rank=[202, 914, 130506, 65493, 199, 31871, 32838, 121] avg=32768.0000 max=130506 max_rank=2 min=121 min_rank=7 imbalance_ratio=3.982727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=39 tokens_per_ep_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64803 max_load_reduction_pct=0.661937 moved_to_replicas=97171 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=40 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] avg=32768.0000 max=33096 max_rank=1 min=32086 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97410 max_load_reduction_pct=0.746402 moved_to_replicas=129807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] avg=32768.0000 max=33096 max_rank=2 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64902 max_load_reduction_pct=0.662279 moved_to_replicas=97078 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=35 tokens_per_ep_rank=[32900, 32308, 33038, 32653, 32565, 32486, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=32308 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32168 max_load_reduction_pct=0.492875 moved_to_replicas=38010 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=40 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] avg=32768.0000 max=33096 max_rank=1 min=32086 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97410 max_load_reduction_pct=0.746402 moved_to_replicas=129807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=40 tokens_per_ep_rank=[202, 914, 130506, 65493, 199, 31871, 32838, 121] avg=32768.0000 max=130506 max_rank=2 min=121 min_rank=7 imbalance_ratio=3.982727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 97998, 65272, 32749, 19, 430, 185] avg=32768.0000 max=97998 max_rank=2 min=19 min_rank=5 imbalance_ratio=2.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=38 tokens_per_ep_rank=[32755, 65429, 32675, 65582, 259, 2, 252, 65190] avg=32768.0000 max=65582 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.001404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 97998, 65272, 32749, 19, 430, 185] avg=32768.0000 max=97998 max_rank=2 min=19 min_rank=5 imbalance_ratio=2.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=40 tokens_per_ep_rank=[202, 914, 130506, 65493, 199, 31871, 32838, 121] avg=32768.0000 max=130506 max_rank=2 min=121 min_rank=7 imbalance_ratio=3.982727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 135, 32897, 184, 97972, 32705] avg=32768.0000 max=97972 max_rank=6 min=135 min_rank=3 imbalance_ratio=2.989868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=40 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] avg=32768.0000 max=33096 max_rank=1 min=32086 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97410 max_load_reduction_pct=0.746402 moved_to_replicas=129807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] avg=32768.0000 max=33096 max_rank=2 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64902 max_load_reduction_pct=0.662279 moved_to_replicas=97078 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=38 tokens_per_ep_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32486 max_load_reduction_pct=0.495349 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 97998, 65272, 32749, 19, 430, 185] avg=32768.0000 max=97998 max_rank=2 min=19 min_rank=5 imbalance_ratio=2.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] avg=32768.0000 max=33096 max_rank=2 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64902 max_load_reduction_pct=0.662279 moved_to_replicas=97078 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=40 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] avg=32768.0000 max=33096 max_rank=1 min=32086 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97410 max_load_reduction_pct=0.746402 moved_to_replicas=129807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=36 tokens_per_ep_rank=[185, 163216, 152, 32664, 136, 32875, 276, 32640] avg=32768.0000 max=163216 max_rank=1 min=136 min_rank=4 imbalance_ratio=4.980957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] avg=32768.0000 max=33096 max_rank=6 min=32426 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64876 max_load_reduction_pct=0.662189 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 97998, 65272, 32749, 19, 430, 185] avg=32768.0000 max=97998 max_rank=2 min=19 min_rank=5 imbalance_ratio=2.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] avg=32768.0000 max=33096 max_rank=2 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64902 max_load_reduction_pct=0.662279 moved_to_replicas=97078 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 135, 32897, 184, 97972, 32705] avg=32768.0000 max=97972 max_rank=6 min=135 min_rank=3 imbalance_ratio=2.989868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=39 tokens_per_ep_rank=[32655, 97899, 32714, 388, 254, 65464, 32762, 8] avg=32768.0000 max=97899 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.987640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=36 tokens_per_ep_rank=[32801, 33096, 32784, 32664, 32774, 32875, 32510, 32640] avg=32768.0000 max=33096 max_rank=1 min=32510 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130120 max_load_reduction_pct=0.797226 moved_to_replicas=130120 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 97998, 65272, 32749, 19, 430, 185] avg=32768.0000 max=97998 max_rank=2 min=19 min_rank=5 imbalance_ratio=2.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 135, 32897, 184, 97972, 32705] avg=32768.0000 max=97972 max_rank=6 min=135 min_rank=3 imbalance_ratio=2.989868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] avg=32768.0000 max=33096 max_rank=2 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64902 max_load_reduction_pct=0.662279 moved_to_replicas=97078 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=43 tokens_per_ep_rank=[99, 98197, 134, 261, 32592, 65, 32625, 98171] avg=32768.0000 max=98197 max_rank=1 min=65 min_rank=5 imbalance_ratio=2.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] avg=32768.0000 max=33096 max_rank=6 min=32426 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64876 max_load_reduction_pct=0.662189 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=39 tokens_per_ep_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64803 max_load_reduction_pct=0.661937 moved_to_replicas=97171 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 135, 32897, 184, 97972, 32705] avg=32768.0000 max=97972 max_rank=6 min=135 min_rank=3 imbalance_ratio=2.989868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] avg=32768.0000 max=33096 max_rank=2 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64902 max_load_reduction_pct=0.662279 moved_to_replicas=97078 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] avg=32768.0000 max=33096 max_rank=6 min=32426 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64876 max_load_reduction_pct=0.662189 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=43 tokens_per_ep_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] avg=32768.0000 max=33096 max_rank=1 min=32563 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65101 max_load_reduction_pct=0.662963 moved_to_replicas=130176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=37 tokens_per_ep_rank=[32639, 3194, 268, 95217, 32899, 32467, 57, 65403] avg=32768.0000 max=95217 max_rank=3 min=57 min_rank=6 imbalance_ratio=2.905792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 135, 32897, 184, 97972, 32705] avg=32768.0000 max=97972 max_rank=6 min=135 min_rank=3 imbalance_ratio=2.989868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] avg=32768.0000 max=33096 max_rank=6 min=32426 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64876 max_load_reduction_pct=0.662189 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=43 tokens_per_ep_rank=[99, 98197, 134, 261, 32592, 65, 32625, 98171] avg=32768.0000 max=98197 max_rank=1 min=65 min_rank=5 imbalance_ratio=2.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=40 tokens_per_ep_rank=[202, 914, 130506, 65493, 199, 31871, 32838, 121] avg=32768.0000 max=130506 max_rank=2 min=121 min_rank=7 imbalance_ratio=3.982727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=37 tokens_per_ep_rank=[32639, 33096, 32162, 33096, 32899, 32467, 32689, 33096] avg=32768.0000 max=33096 max_rank=1 min=32162 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62121 max_load_reduction_pct=0.652415 moved_to_replicas=94428 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 135, 32897, 184, 97972, 32705] avg=32768.0000 max=97972 max_rank=6 min=135 min_rank=3 imbalance_ratio=2.989868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=43 tokens_per_ep_rank=[99, 98197, 134, 261, 32592, 65, 32625, 98171] avg=32768.0000 max=98197 max_rank=1 min=65 min_rank=5 imbalance_ratio=2.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] avg=32768.0000 max=33096 max_rank=6 min=32426 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64876 max_load_reduction_pct=0.662189 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=44 tokens_per_ep_rank=[31468, 303, 33, 32773, 32755, 32690, 71540, 60582] avg=32768.0000 max=71540 max_rank=6 min=33 min_rank=2 imbalance_ratio=2.183228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=43 tokens_per_ep_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] avg=32768.0000 max=33096 max_rank=1 min=32563 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65101 max_load_reduction_pct=0.662963 moved_to_replicas=130176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=43 tokens_per_ep_rank=[99, 98197, 134, 261, 32592, 65, 32625, 98171] avg=32768.0000 max=98197 max_rank=1 min=65 min_rank=5 imbalance_ratio=2.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=40 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] avg=32768.0000 max=33096 max_rank=1 min=32086 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97410 max_load_reduction_pct=0.746402 moved_to_replicas=129807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] avg=32768.0000 max=33096 max_rank=6 min=32426 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64876 max_load_reduction_pct=0.662189 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=43 tokens_per_ep_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] avg=32768.0000 max=33096 max_rank=1 min=32563 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65101 max_load_reduction_pct=0.662963 moved_to_replicas=130176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=44 tokens_per_ep_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=31968 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=38442 max_load_reduction_pct=0.537350 moved_to_replicas=65930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=38 tokens_per_ep_rank=[32755, 65429, 32675, 65582, 259, 2, 252, 65190] avg=32768.0000 max=65582 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.001404 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=43 tokens_per_ep_rank=[99, 98197, 134, 261, 32592, 65, 32625, 98171] avg=32768.0000 max=98197 max_rank=1 min=65 min_rank=5 imbalance_ratio=2.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=43 tokens_per_ep_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] avg=32768.0000 max=33096 max_rank=1 min=32563 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65101 max_load_reduction_pct=0.662963 moved_to_replicas=130176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=44 tokens_per_ep_rank=[31468, 303, 33, 32773, 32755, 32690, 71540, 60582] avg=32768.0000 max=71540 max_rank=6 min=33 min_rank=2 imbalance_ratio=2.183228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 97998, 65272, 32749, 19, 430, 185] avg=32768.0000 max=97998 max_rank=2 min=19 min_rank=5 imbalance_ratio=2.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=43 tokens_per_ep_rank=[99, 98197, 134, 261, 32592, 65, 32625, 98171] avg=32768.0000 max=98197 max_rank=1 min=65 min_rank=5 imbalance_ratio=2.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=38 tokens_per_ep_rank=[32755, 33096, 32675, 33096, 32353, 32488, 32585, 33096] avg=32768.0000 max=33096 max_rank=1 min=32353 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32486 max_load_reduction_pct=0.495349 moved_to_replicas=96913 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=44 tokens_per_ep_rank=[31468, 303, 33, 32773, 32755, 32690, 71540, 60582] avg=32768.0000 max=71540 max_rank=6 min=33 min_rank=2 imbalance_ratio=2.183228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=43 tokens_per_ep_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] avg=32768.0000 max=33096 max_rank=1 min=32563 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65101 max_load_reduction_pct=0.662963 moved_to_replicas=130176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 673, 33, 65338, 97913, 284, 32810] avg=32768.0000 max=97913 max_rank=5 min=33 min_rank=3 imbalance_ratio=2.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=44 tokens_per_ep_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=31968 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=38442 max_load_reduction_pct=0.537350 moved_to_replicas=65930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=44 tokens_per_ep_rank=[31468, 303, 33, 32773, 32755, 32690, 71540, 60582] avg=32768.0000 max=71540 max_rank=6 min=33 min_rank=2 imbalance_ratio=2.183228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] avg=32768.0000 max=33096 max_rank=2 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64902 max_load_reduction_pct=0.662279 moved_to_replicas=97078 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=43 tokens_per_ep_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] avg=32768.0000 max=33096 max_rank=1 min=32563 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65101 max_load_reduction_pct=0.662963 moved_to_replicas=130176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=44 tokens_per_ep_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=31968 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=38442 max_load_reduction_pct=0.537350 moved_to_replicas=65930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] avg=32768.0000 max=33096 max_rank=4 min=32262 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64817 max_load_reduction_pct=0.661986 moved_to_replicas=97059 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=39 tokens_per_ep_rank=[32655, 97899, 32714, 388, 254, 65464, 32762, 8] avg=32768.0000 max=97899 max_rank=1 min=8 min_rank=7 imbalance_ratio=2.987640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=44 tokens_per_ep_rank=[31468, 303, 33, 32773, 32755, 32690, 71540, 60582] avg=32768.0000 max=71540 max_rank=6 min=33 min_rank=2 imbalance_ratio=2.183228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=44 tokens_per_ep_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=31968 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=38442 max_load_reduction_pct=0.537350 moved_to_replicas=65930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 673, 33, 65338, 97913, 284, 32810] avg=32768.0000 max=97913 max_rank=5 min=33 min_rank=3 imbalance_ratio=2.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 135, 32897, 184, 97972, 32705] avg=32768.0000 max=97972 max_rank=6 min=135 min_rank=3 imbalance_ratio=2.989868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=44 tokens_per_ep_rank=[31468, 303, 33, 32773, 32755, 32690, 71540, 60582] avg=32768.0000 max=71540 max_rank=6 min=33 min_rank=2 imbalance_ratio=2.183228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=39 tokens_per_ep_rank=[32655, 33096, 32714, 32756, 32421, 33096, 32762, 32644] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64803 max_load_reduction_pct=0.661937 moved_to_replicas=97171 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=44 tokens_per_ep_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=31968 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=38442 max_load_reduction_pct=0.537350 moved_to_replicas=65930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 673, 33, 65338, 97913, 284, 32810] avg=32768.0000 max=97913 max_rank=5 min=33 min_rank=3 imbalance_ratio=2.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=46 tokens_per_ep_rank=[32476, 208, 32892, 175, 255, 65456, 65400, 65282] avg=32768.0000 max=65456 max_rank=5 min=175 min_rank=3 imbalance_ratio=1.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] avg=32768.0000 max=33096 max_rank=4 min=32262 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64817 max_load_reduction_pct=0.661986 moved_to_replicas=97059 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 673, 33, 65338, 97913, 284, 32810] avg=32768.0000 max=97913 max_rank=5 min=33 min_rank=3 imbalance_ratio=2.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=44 tokens_per_ep_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=31968 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=38442 max_load_reduction_pct=0.537350 moved_to_replicas=65930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] avg=32768.0000 max=33096 max_rank=6 min=32426 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64876 max_load_reduction_pct=0.662189 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] avg=32768.0000 max=33096 max_rank=4 min=32262 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64817 max_load_reduction_pct=0.661986 moved_to_replicas=97059 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=46 tokens_per_ep_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32360 max_load_reduction_pct=0.494378 moved_to_replicas=96850 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=40 tokens_per_ep_rank=[202, 914, 130506, 65493, 199, 31871, 32838, 121] avg=32768.0000 max=130506 max_rank=2 min=121 min_rank=7 imbalance_ratio=3.982727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 673, 33, 65338, 97913, 284, 32810] avg=32768.0000 max=97913 max_rank=5 min=33 min_rank=3 imbalance_ratio=2.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=46 tokens_per_ep_rank=[32476, 208, 32892, 175, 255, 65456, 65400, 65282] avg=32768.0000 max=65456 max_rank=5 min=175 min_rank=3 imbalance_ratio=1.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] avg=32768.0000 max=33096 max_rank=4 min=32262 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64817 max_load_reduction_pct=0.661986 moved_to_replicas=97059 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=40 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32835, 32086, 32838, 32757] avg=32768.0000 max=33096 max_rank=1 min=32086 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97410 max_load_reduction_pct=0.746402 moved_to_replicas=129807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 673, 33, 65338, 97913, 284, 32810] avg=32768.0000 max=97913 max_rank=5 min=33 min_rank=3 imbalance_ratio=2.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] avg=32768.0000 max=33096 max_rank=4 min=32262 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64817 max_load_reduction_pct=0.661986 moved_to_replicas=97059 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=46 tokens_per_ep_rank=[32476, 208, 32892, 175, 255, 65456, 65400, 65282] avg=32768.0000 max=65456 max_rank=5 min=175 min_rank=3 imbalance_ratio=1.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=43 tokens_per_ep_rank=[99, 98197, 134, 261, 32592, 65, 32625, 98171] avg=32768.0000 max=98197 max_rank=1 min=65 min_rank=5 imbalance_ratio=2.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=47 tokens_per_ep_rank=[262, 33017, 65274, 32546, 32655, 65400, 96, 32894] avg=32768.0000 max=65400 max_rank=5 min=96 min_rank=6 imbalance_ratio=1.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=46 tokens_per_ep_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32360 max_load_reduction_pct=0.494378 moved_to_replicas=96850 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] avg=32768.0000 max=33096 max_rank=4 min=32262 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64817 max_load_reduction_pct=0.661986 moved_to_replicas=97059 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=46 tokens_per_ep_rank=[32476, 208, 32892, 175, 255, 65456, 65400, 65282] avg=32768.0000 max=65456 max_rank=5 min=175 min_rank=3 imbalance_ratio=1.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=46 tokens_per_ep_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32360 max_load_reduction_pct=0.494378 moved_to_replicas=96850 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=47 tokens_per_ep_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32304 max_load_reduction_pct=0.493945 moved_to_replicas=64482 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=43 tokens_per_ep_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] avg=32768.0000 max=33096 max_rank=1 min=32563 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65101 max_load_reduction_pct=0.662963 moved_to_replicas=130176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 97998, 65272, 32749, 19, 430, 185] avg=32768.0000 max=97998 max_rank=2 min=19 min_rank=5 imbalance_ratio=2.990662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=46 tokens_per_ep_rank=[32476, 208, 32892, 175, 255, 65456, 65400, 65282] avg=32768.0000 max=65456 max_rank=5 min=175 min_rank=3 imbalance_ratio=1.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=47 tokens_per_ep_rank=[262, 33017, 65274, 32546, 32655, 65400, 96, 32894] avg=32768.0000 max=65400 max_rank=5 min=96 min_rank=6 imbalance_ratio=1.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=46 tokens_per_ep_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32360 max_load_reduction_pct=0.494378 moved_to_replicas=96850 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=46 tokens_per_ep_rank=[32476, 208, 32892, 175, 255, 65456, 65400, 65282] avg=32768.0000 max=65456 max_rank=5 min=175 min_rank=3 imbalance_ratio=1.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=41 tokens_per_ep_rank=[32700, 32791, 33096, 33096, 32749, 32643, 32606, 32463] avg=32768.0000 max=33096 max_rank=2 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64902 max_load_reduction_pct=0.662279 moved_to_replicas=97078 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=46 tokens_per_ep_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32360 max_load_reduction_pct=0.494378 moved_to_replicas=96850 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=47 tokens_per_ep_rank=[262, 33017, 65274, 32546, 32655, 65400, 96, 32894] avg=32768.0000 max=65400 max_rank=5 min=96 min_rank=6 imbalance_ratio=1.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=47 tokens_per_ep_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32304 max_load_reduction_pct=0.493945 moved_to_replicas=64482 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=48 tokens_per_ep_rank=[164, 65277, 32577, 65454, 32709, 32955, 32646, 362] avg=32768.0000 max=65454 max_rank=3 min=164 min_rank=0 imbalance_ratio=1.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=44 tokens_per_ep_rank=[31468, 303, 33, 32773, 32755, 32690, 71540, 60582] avg=32768.0000 max=71540 max_rank=6 min=33 min_rank=2 imbalance_ratio=2.183228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=47 tokens_per_ep_rank=[262, 33017, 65274, 32546, 32655, 65400, 96, 32894] avg=32768.0000 max=65400 max_rank=5 min=96 min_rank=6 imbalance_ratio=1.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=46 tokens_per_ep_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32360 max_load_reduction_pct=0.494378 moved_to_replicas=96850 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=47 tokens_per_ep_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32304 max_load_reduction_pct=0.493945 moved_to_replicas=64482 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=48 tokens_per_ep_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] avg=32768.0000 max=33096 max_rank=1 min=32522 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32358 max_load_reduction_pct=0.494362 moved_to_replicas=64539 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=44 tokens_per_ep_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=31968 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=38442 max_load_reduction_pct=0.537350 moved_to_replicas=65930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 135, 32897, 184, 97972, 32705] avg=32768.0000 max=97972 max_rank=6 min=135 min_rank=3 imbalance_ratio=2.989868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=47 tokens_per_ep_rank=[262, 33017, 65274, 32546, 32655, 65400, 96, 32894] avg=32768.0000 max=65400 max_rank=5 min=96 min_rank=6 imbalance_ratio=1.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=48 tokens_per_ep_rank=[164, 65277, 32577, 65454, 32709, 32955, 32646, 362] avg=32768.0000 max=65454 max_rank=3 min=164 min_rank=0 imbalance_ratio=1.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=47 tokens_per_ep_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32304 max_load_reduction_pct=0.493945 moved_to_replicas=64482 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=47 tokens_per_ep_rank=[262, 33017, 65274, 32546, 32655, 65400, 96, 32894] avg=32768.0000 max=65400 max_rank=5 min=96 min_rank=6 imbalance_ratio=1.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=48 tokens_per_ep_rank=[164, 65277, 32577, 65454, 32709, 32955, 32646, 362] avg=32768.0000 max=65454 max_rank=3 min=164 min_rank=0 imbalance_ratio=1.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=47 tokens_per_ep_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32304 max_load_reduction_pct=0.493945 moved_to_replicas=64482 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=42 tokens_per_ep_rank=[32760, 32767, 32724, 32769, 32897, 32426, 33096, 32705] avg=32768.0000 max=33096 max_rank=6 min=32426 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64876 max_load_reduction_pct=0.662189 moved_to_replicas=64876 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=48 tokens_per_ep_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] avg=32768.0000 max=33096 max_rank=1 min=32522 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32358 max_load_reduction_pct=0.494362 moved_to_replicas=64539 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 673, 33, 65338, 97913, 284, 32810] avg=32768.0000 max=97913 max_rank=5 min=33 min_rank=3 imbalance_ratio=2.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=48 tokens_per_ep_rank=[164, 65277, 32577, 65454, 32709, 32955, 32646, 362] avg=32768.0000 max=65454 max_rank=3 min=164 min_rank=0 imbalance_ratio=1.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=47 tokens_per_ep_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32304 max_load_reduction_pct=0.493945 moved_to_replicas=64482 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=48 tokens_per_ep_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] avg=32768.0000 max=33096 max_rank=1 min=32522 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32358 max_load_reduction_pct=0.494362 moved_to_replicas=64539 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] avg=32768.0000 max=33096 max_rank=4 min=32262 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64817 max_load_reduction_pct=0.661986 moved_to_replicas=97059 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=48 tokens_per_ep_rank=[164, 65277, 32577, 65454, 32709, 32955, 32646, 362] avg=32768.0000 max=65454 max_rank=3 min=164 min_rank=0 imbalance_ratio=1.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=48 tokens_per_ep_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] avg=32768.0000 max=33096 max_rank=1 min=32522 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32358 max_load_reduction_pct=0.494362 moved_to_replicas=64539 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=43 tokens_per_ep_rank=[99, 98197, 134, 261, 32592, 65, 32625, 98171] avg=32768.0000 max=98197 max_rank=1 min=65 min_rank=5 imbalance_ratio=2.996735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=48 tokens_per_ep_rank=[164, 65277, 32577, 65454, 32709, 32955, 32646, 362] avg=32768.0000 max=65454 max_rank=3 min=164 min_rank=0 imbalance_ratio=1.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=48 tokens_per_ep_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] avg=32768.0000 max=33096 max_rank=1 min=32522 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32358 max_load_reduction_pct=0.494362 moved_to_replicas=64539 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=43 tokens_per_ep_rank=[32563, 33096, 32771, 32699, 32592, 32702, 32625, 33096] avg=32768.0000 max=33096 max_rank=1 min=32563 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65101 max_load_reduction_pct=0.662963 moved_to_replicas=130176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=46 tokens_per_ep_rank=[32476, 208, 32892, 175, 255, 65456, 65400, 65282] avg=32768.0000 max=65456 max_rank=5 min=175 min_rank=3 imbalance_ratio=1.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=48 tokens_per_ep_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] avg=32768.0000 max=33096 max_rank=1 min=32522 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32358 max_load_reduction_pct=0.494362 moved_to_replicas=64539 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=46 tokens_per_ep_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32360 max_load_reduction_pct=0.494378 moved_to_replicas=96850 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=44 tokens_per_ep_rank=[31468, 303, 33, 32773, 32755, 32690, 71540, 60582] avg=32768.0000 max=71540 max_rank=6 min=33 min_rank=2 imbalance_ratio=2.183228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=44 tokens_per_ep_rank=[31968, 33095, 32670, 32773, 32755, 32690, 33095, 33098] avg=32768.0000 max=33098 max_rank=7 min=31968 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=38442 max_load_reduction_pct=0.537350 moved_to_replicas=65930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=47 tokens_per_ep_rank=[262, 33017, 65274, 32546, 32655, 65400, 96, 32894] avg=32768.0000 max=65400 max_rank=5 min=96 min_rank=6 imbalance_ratio=1.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=47 tokens_per_ep_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32304 max_load_reduction_pct=0.493945 moved_to_replicas=64482 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 673, 33, 65338, 97913, 284, 32810] avg=32768.0000 max=97913 max_rank=5 min=33 min_rank=3 imbalance_ratio=2.988068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=45 tokens_per_ep_rank=[32831, 32262, 32915, 32670, 33096, 33096, 32464, 32810] avg=32768.0000 max=33096 max_rank=4 min=32262 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64817 max_load_reduction_pct=0.661986 moved_to_replicas=97059 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=48 tokens_per_ep_rank=[164, 65277, 32577, 65454, 32709, 32955, 32646, 362] avg=32768.0000 max=65454 max_rank=3 min=164 min_rank=0 imbalance_ratio=1.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=48 tokens_per_ep_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] avg=32768.0000 max=33096 max_rank=1 min=32522 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32358 max_load_reduction_pct=0.494362 moved_to_replicas=64539 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=46 tokens_per_ep_rank=[32476, 208, 32892, 175, 255, 65456, 65400, 65282] avg=32768.0000 max=65456 max_rank=5 min=175 min_rank=3 imbalance_ratio=1.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=46 tokens_per_ep_rank=[32476, 32512, 32892, 32535, 32441, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32360 max_load_reduction_pct=0.494378 moved_to_replicas=96850 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=47 tokens_per_ep_rank=[262, 33017, 65274, 32546, 32655, 65400, 96, 32894] avg=32768.0000 max=65400 max_rank=5 min=96 min_rank=6 imbalance_ratio=1.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=47 tokens_per_ep_rank=[32440, 33017, 33096, 32546, 32655, 33096, 32400, 32894] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32304 max_load_reduction_pct=0.493945 moved_to_replicas=64482 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=48 tokens_per_ep_rank=[164, 65277, 32577, 65454, 32709, 32955, 32646, 362] avg=32768.0000 max=65454 max_rank=3 min=164 min_rank=0 imbalance_ratio=1.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=48 tokens_per_ep_rank=[32522, 33096, 32577, 33096, 32709, 32955, 32646, 32543] avg=32768.0000 max=33096 max_rank=1 min=32522 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32358 max_load_reduction_pct=0.494362 moved_to_replicas=64539 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=48 total_expected_recv=1572406 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366336 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=48 total_expected_recv=1572208 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=367104 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=48 total_expected_recv=1572504 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366080 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=48 total_expected_recv=1570691 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=369408 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=48 total_expected_recv=1572289 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365312 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=48 total_expected_recv=1577937 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=358144 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=48 total_expected_recv=1575003 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364288 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=48 total_expected_recv=1569874 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368896 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:47:02] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 713.3 | throughput per GPU (TFLOP/s/GPU): 124.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.292158E+00 | loss scale: 1.0 | grad norm: 6.938 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=30575 expected_overflow=0 exact_recv_per_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=30575 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34816 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=31866 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32109 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=31866 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32109 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=31165 expected_overflow=0 exact_recv_per_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=31165 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=31329 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32238 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=31329 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32238 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34816 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=31864 expected_overflow=0 exact_recv_per_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=31660 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33046 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=31660 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33046 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=31040 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=31864 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=31838 expected_overflow=0 exact_recv_per_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=31838 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32945 expected_overflow=0 exact_recv_per_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32945 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=31555 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=31555 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=31040 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32413 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=31080 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=31080 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32301 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32301 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32413 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32137 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32407 expected_overflow=0 exact_recv_per_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32137 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32407 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32283 expected_overflow=0 exact_recv_per_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32041 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32041 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32283 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32251 expected_overflow=0 exact_recv_per_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32251 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32364 expected_overflow=0 exact_recv_per_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32364 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=31515 expected_overflow=0 exact_recv_per_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=31515 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=31410 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=31410 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33054 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33054 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32395 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32152 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32395 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32152 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33086 expected_overflow=0 exact_recv_per_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32346 expected_overflow=0 exact_recv_per_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33086 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32346 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=40960 expected_recv=31921 expected_overflow=0 exact_recv_per_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=40960 expected_recv=31921 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=40960 expected_recv=31506 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=40960 expected_recv=31506 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32161 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32161 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32320 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32452 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32452 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=40960 expected_recv=31569 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=40960 expected_recv=31569 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32046 expected_overflow=0 exact_recv_per_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32046 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=40960 expected_recv=31759 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=40960 expected_recv=31759 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32489 expected_overflow=0 exact_recv_per_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32489 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32454 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32454 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32359 expected_overflow=0 exact_recv_per_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32359 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32461 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32461 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32085 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32085 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=40960 expected_recv=31702 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=40960 expected_recv=31702 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32104 expected_overflow=0 exact_recv_per_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32104 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32238 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32238 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=40960 expected_recv=31812 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=40960 expected_recv=31812 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32389 expected_overflow=0 exact_recv_per_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32389 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=40960 expected_recv=31129 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=23 calls=1 budget=40960 expected_recv=31523 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=40960 expected_recv=31129 actual_valid_recv=31232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=23 calls=1 budget=40960 expected_recv=31523 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=40960 expected_recv=31753 expected_overflow=0 exact_recv_per_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=40960 expected_recv=31753 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32124 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32124 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=25 calls=1 budget=40960 expected_recv=31931 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=25 calls=1 budget=40960 expected_recv=31931 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33020 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33020 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32052 expected_overflow=0 exact_recv_per_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32416 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32052 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32416 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32218 expected_overflow=0 exact_recv_per_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32218 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32410 expected_overflow=0 exact_recv_per_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32410 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32082 expected_overflow=0 exact_recv_per_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32082 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32366 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32392 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32366 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32392 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32455 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32462 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32462 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=27 calls=1 budget=40960 expected_recv=31796 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=27 calls=1 budget=40960 expected_recv=31796 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32047 expected_overflow=0 exact_recv_per_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32047 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33021 expected_overflow=0 exact_recv_per_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=27 calls=1 budget=40960 expected_recv=33021 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32424 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32424 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32078 expected_overflow=0 exact_recv_per_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32078 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32340 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=28 calls=1 budget=40960 expected_recv=31715 expected_overflow=0 exact_recv_per_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32340 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=28 calls=1 budget=40960 expected_recv=31715 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=29 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32397 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=29 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32397 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=30 calls=1 budget=40960 expected_recv=31512 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=30 calls=1 budget=40960 expected_recv=31512 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32344 expected_overflow=0 exact_recv_per_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32344 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32278 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32503 expected_overflow=0 exact_recv_per_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32416 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32278 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32503 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=32 calls=1 budget=40960 expected_recv=31875 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32416 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=32 calls=1 budget=40960 expected_recv=31875 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32394 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32394 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32348 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32348 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=34 calls=1 budget=40960 expected_recv=31760 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=34 calls=1 budget=40960 expected_recv=31760 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32255 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32255 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=35 calls=1 budget=40960 expected_recv=31846 expected_overflow=0 exact_recv_per_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=35 calls=1 budget=40960 expected_recv=31846 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32312 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32312 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32205 expected_overflow=0 exact_recv_per_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32412 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32205 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32077 expected_overflow=0 exact_recv_per_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32077 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=38 calls=1 budget=40960 expected_recv=31788 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=38 calls=1 budget=40960 expected_recv=31788 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32117 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32117 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=42 calls=1 budget=40960 expected_recv=31414 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=42 calls=1 budget=40960 expected_recv=31414 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32511 expected_overflow=0 exact_recv_per_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32511 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32461 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32461 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32328 expected_overflow=0 exact_recv_per_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32328 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32106 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32106 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=42 calls=1 budget=40960 expected_recv=31130 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=42 calls=1 budget=40960 expected_recv=31130 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32473 expected_overflow=0 exact_recv_per_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32473 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32410 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32410 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=43 calls=1 budget=40960 expected_recv=31911 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=43 calls=1 budget=40960 expected_recv=31911 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32462 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32462 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32174 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32402 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32402 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[26215, 18949, 41541, 32583, 38787, 29862, 36748, 37459] avg=32768.0000 max=41541 max_rank=2 min=18949 min_rank=1 imbalance_ratio=1.267731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[31907, 32174, 33096, 32583, 33095, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31907 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8444 max_load_reduction_pct=0.203269 moved_to_replicas=22151 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[38407, 85816, 262, 76, 74259, 4555, 14176, 44593] avg=32768.0000 max=85816 max_rank=1 min=76 min_rank=3 imbalance_ratio=2.618896 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 31866, 32689, 33096, 33096, 32109, 33096] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52720 max_load_reduction_pct=0.614338 moved_to_replicas=110691 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[27677, 47773, 11131, 183, 38999, 24072, 58105, 54204] avg=32768.0000 max=58105 max_rank=6 min=183 min_rank=3 imbalance_ratio=1.773224 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33097, 32238, 31329, 33096, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31329 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25008 max_load_reduction_pct=0.430393 moved_to_replicas=66697 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 22808, 843, 32419, 43186, 32981, 65497] avg=32768.0000 max=65497 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.998810 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32750, 31660, 33046, 33096, 32419, 33096, 32981, 33096] avg=32768.0000 max=33096 max_rank=3 min=31660 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32401 max_load_reduction_pct=0.494694 moved_to_replicas=42491 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[39, 41694, 65593, 34245, 59563, 60919, 30, 61] avg=32768.0000 max=65593 max_rank=2 min=30 min_rank=6 imbalance_ratio=2.001740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[31040, 33096, 33096, 33096, 33096, 33097, 32527, 33096] avg=32768.0000 max=33097 max_rank=5 min=31040 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32496 max_load_reduction_pct=0.495419 moved_to_replicas=96534 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[45069, 465, 54232, 835, 32824, 1431, 57, 127231] avg=32768.0000 max=127231 max_rank=7 min=57 min_rank=6 imbalance_ratio=3.882782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[33095, 33096, 33096, 31080, 32824, 33097, 32760, 33096] avg=32768.0000 max=33097 max_rank=5 min=31080 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=94134 max_load_reduction_pct=0.739867 moved_to_replicas=127244 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65495, 32716, 351, 32770, 65046, 204, 28] avg=32768.0000 max=65534 max_rank=0 min=28 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[33096, 33096, 32716, 32301, 32770, 33096, 32603, 32466] avg=32768.0000 max=33096 max_rank=0 min=32301 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96787 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32700, 53, 46, 32531, 65482, 72794, 25741, 32797] avg=32768.0000 max=72794 max_rank=5 min=46 min_rank=2 imbalance_ratio=2.221497 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32700, 33096, 32788, 32531, 33096, 33095, 32041, 32797] avg=32768.0000 max=33096 max_rank=1 min=32041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=39698 max_load_reduction_pct=0.545347 moved_to_replicas=72084 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[98232, 32757, 32729, 32745, 429, 25, 15, 65212] avg=32768.0000 max=98232 max_rank=0 min=15 min_rank=6 imbalance_ratio=2.997803 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32757, 32729, 32745, 32545, 32419, 32757, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=97252 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[0, 65, 17, 45, 33720, 98216, 32782, 97299] avg=32768.0000 max=98216 max_rank=5 min=0 min_rank=0 imbalance_ratio=2.997314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32742, 32152, 32395, 32785, 33096, 33096, 32782, 33096] avg=32768.0000 max=33096 max_rank=4 min=32152 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65120 max_load_reduction_pct=0.663028 moved_to_replicas=129947 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[19, 32745, 65435, 33571, 32854, 32140, 32549, 32831] avg=32768.0000 max=65435 max_rank=2 min=19 min_rank=0 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32358, 32745, 33096, 33096, 32854, 32615, 32549, 32831] avg=32768.0000 max=33096 max_rank=2 min=32358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=32814 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[33398, 61867, 45, 6, 74037, 32770, 27285, 32736] avg=32768.0000 max=74037 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.259430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32748, 33096, 32770, 31506, 32736] avg=32768.0000 max=33096 max_rank=0 min=31506 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40941 max_load_reduction_pct=0.552980 moved_to_replicas=70014 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[64692, 27, 33724, 32743, 32769, 32723, 236, 65230] avg=32768.0000 max=65230 max_rank=7 min=27 min_rank=1 imbalance_ratio=1.990662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[33096, 32161, 33096, 32743, 32769, 32723, 32460, 33096] avg=32768.0000 max=33096 max_rank=0 min=32161 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.492626 moved_to_replicas=64358 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[65486, 604, 28253, 65560, 46651, 55349, 214, 27] avg=32768.0000 max=65560 max_rank=3 min=27 min_rank=7 imbalance_ratio=2.000732 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[33096, 33097, 31569, 33096, 33094, 33097, 32604, 32491] avg=32768.0000 max=33097 max_rank=1 min=31569 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32463 max_load_reduction_pct=0.495165 moved_to_replicas=100662 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[65518, 98252, 29, 32784, 4, 32700, 90, 32767] avg=32768.0000 max=98252 max_rank=1 min=4 min_rank=4 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[33096, 33096, 32443, 32784, 32746, 32700, 32512, 32767] avg=32768.0000 max=33096 max_rank=0 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=97578 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32742, 0, 35990, 29566, 32775, 65550, 32778, 32743] avg=32768.0000 max=65550 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000427 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32742, 32454, 33096, 32460, 32775, 33096, 32778, 32743] avg=32768.0000 max=33096 max_rank=2 min=32454 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32454 max_load_reduction_pct=0.495103 moved_to_replicas=35348 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[32768, 56, 32758, 32821, 98210, 10, 32759, 32762] avg=32768.0000 max=98210 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.997131 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[32768, 32429, 32758, 32821, 33096, 32751, 32759, 32762] avg=32768.0000 max=33096 max_rank=4 min=32429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65114 max_load_reduction_pct=0.663008 moved_to_replicas=65114 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[33268, 48019, 50, 65478, 110, 32768, 65461, 16990] avg=32768.0000 max=65478 max_rank=3 min=50 min_rank=2 imbalance_ratio=1.998230 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[33096, 33096, 32432, 33096, 32475, 32768, 33096, 32085] avg=32768.0000 max=33096 max_rank=0 min=32085 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32382 max_load_reduction_pct=0.494548 moved_to_replicas=79842 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[24, 78768, 65546, 50, 33652, 3, 32815, 51286] avg=32768.0000 max=78768 max_rank=1 min=3 min_rank=5 imbalance_ratio=2.403809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[31702, 33096, 33096, 32500, 33096, 32745, 32815, 33094] avg=32768.0000 max=33096 max_rank=1 min=31702 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45672 max_load_reduction_pct=0.579829 moved_to_replicas=96868 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[32793, 65555, 2, 32817, 28, 32425, 65613, 32911] avg=32768.0000 max=65613 max_rank=6 min=2 min_rank=2 imbalance_ratio=2.002350 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[32793, 33096, 32519, 32817, 32487, 32425, 33096, 32911] avg=32768.0000 max=33096 max_rank=1 min=32425 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32517 max_load_reduction_pct=0.495588 moved_to_replicas=64976 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[3947, 65306, 32421, 28, 62159, 32743, 32768, 32772] avg=32768.0000 max=65306 max_rank=1 min=28 min_rank=3 imbalance_ratio=1.992981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[33010, 33096, 32421, 32238, 33096, 32743, 32768, 32772] avg=32768.0000 max=33096 max_rank=1 min=32238 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32210 max_load_reduction_pct=0.493217 moved_to_replicas=61273 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[72, 72989, 32742, 89787, 906, 32745, 32743, 160] avg=32768.0000 max=89787 max_rank=3 min=72 min_rank=0 imbalance_ratio=2.740082 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[32814, 33096, 32742, 33096, 33096, 32745, 32743, 31812] avg=32768.0000 max=33096 max_rank=1 min=31812 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56691 max_load_reduction_pct=0.631394 moved_to_replicas=96584 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[0, 65511, 50, 49, 11095, 57347, 19808, 108284] avg=32768.0000 max=108284 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.304565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[32742, 33096, 33096, 32791, 31129, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31129 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=75187 max_load_reduction_pct=0.694350 moved_to_replicas=131854 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[25, 0, 65478, 10, 33244, 32740, 121593, 9054] avg=32768.0000 max=121593 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.710724 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[31523, 32742, 33096, 32752, 33096, 32740, 33097, 33098] avg=32768.0000 max=33098 max_rank=7 min=31523 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=88495 max_load_reduction_pct=0.727797 moved_to_replicas=121027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=25 tokens_per_ep_rank=[18, 33579, 100464, 32802, 32742, 32775, 19, 29745] avg=32768.0000 max=100464 max_rank=2 min=18 min_rank=0 imbalance_ratio=3.065918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=25 tokens_per_ep_rank=[32760, 33096, 33096, 32802, 32742, 32775, 32749, 32124] avg=32768.0000 max=33096 max_rank=1 min=32124 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67368 max_load_reduction_pct=0.670569 moved_to_replicas=67851 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=26 tokens_per_ep_rank=[65488, 32769, 32744, 33020, 33128, 64995, 0, 0] avg=32768.0000 max=65488 max_rank=0 min=0 min_rank=6 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=26 tokens_per_ep_rank=[33096, 32769, 32744, 33020, 33096, 33096, 32392, 31931] avg=32768.0000 max=33096 max_rank=0 min=31931 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64323 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=27 tokens_per_ep_rank=[72, 32774, 39283, 32742, 32794, 43, 124413, 23] avg=32768.0000 max=124413 max_rank=6 min=23 min_rank=7 imbalance_ratio=3.796783 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=27 tokens_per_ep_rank=[32539, 32774, 33096, 32742, 32794, 32366, 33096, 32737] avg=32768.0000 max=33096 max_rank=2 min=32366 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=91317 max_load_reduction_pct=0.733983 moved_to_replicas=97504 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=28 tokens_per_ep_rank=[90867, 65486, 65536, 25, 32704, 54, 28, 7444] avg=32768.0000 max=90867 max_rank=0 min=25 min_rank=3 imbalance_ratio=2.773041 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 32767, 32704, 32494, 31796, 33096] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57771 max_load_reduction_pct=0.635775 moved_to_replicas=122601 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=29 tokens_per_ep_rank=[50, 98254, 65508, 28, 65469, 32745, 51, 39] avg=32768.0000 max=98254 max_rank=1 min=28 min_rank=3 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=29 tokens_per_ep_rank=[32462, 33096, 33096, 32770, 33096, 32745, 32424, 32455] avg=32768.0000 max=33096 max_rank=1 min=32424 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=129943 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=30 tokens_per_ep_rank=[32791, 86623, 32760, 32793, 32778, 11588, 26, 32785] avg=32768.0000 max=86623 max_rank=1 min=26 min_rank=6 imbalance_ratio=2.643524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=30 tokens_per_ep_rank=[32791, 33095, 32760, 32793, 32778, 32375, 32767, 32785] avg=32768.0000 max=33095 max_rank=1 min=32375 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=53528 max_load_reduction_pct=0.617942 moved_to_replicas=53527 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=31 tokens_per_ep_rank=[65561, 76911, 32757, 27, 32752, 2, 20742, 33392] avg=32768.0000 max=76911 max_rank=1 min=2 min_rank=5 imbalance_ratio=2.347137 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=31 tokens_per_ep_rank=[33096, 33096, 32757, 33095, 32752, 32740, 31512, 33096] avg=32768.0000 max=33096 max_rank=0 min=31512 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=43815 max_load_reduction_pct=0.569684 moved_to_replicas=76576 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 65448, 65493, 64, 32791, 32846, 0] avg=32768.0000 max=65493 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.998688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=32 tokens_per_ep_rank=[32746, 32756, 33096, 33096, 32416, 32791, 32846, 32397] avg=32768.0000 max=33096 max_rank=2 min=32397 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32397 max_load_reduction_pct=0.494664 moved_to_replicas=64749 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=33 tokens_per_ep_rank=[63855, 47125, 32743, 32768, 32753, 31875, 484, 20541] avg=32768.0000 max=63855 max_rank=0 min=484 min_rank=6 imbalance_ratio=1.948700 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=33 tokens_per_ep_rank=[33096, 33097, 32743, 32768, 32753, 31875, 32717, 33095] avg=32768.0000 max=33097 max_rank=1 min=31875 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=30758 max_load_reduction_pct=0.481685 moved_to_replicas=44788 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=34 tokens_per_ep_rank=[64794, 32793, 0, 25, 65603, 650, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=2 imbalance_ratio=2.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=34 tokens_per_ep_rank=[33096, 32793, 32742, 32532, 33096, 32348, 32441, 33096] avg=32768.0000 max=33096 max_rank=0 min=32348 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=129388 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=35 tokens_per_ep_rank=[75, 25, 32903, 33034, 97326, 19, 98728, 34] avg=32768.0000 max=98728 max_rank=6 min=19 min_rank=5 imbalance_ratio=3.012939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=35 tokens_per_ep_rank=[32817, 32677, 32903, 33034, 33096, 32761, 33096, 31760] avg=32768.0000 max=33096 max_rank=4 min=31760 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65632 max_load_reduction_pct=0.664776 moved_to_replicas=129862 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=36 tokens_per_ep_rank=[32768, 98438, 32750, 52634, 25, 32787, 25, 12717] avg=32768.0000 max=98438 max_rank=1 min=25 min_rank=4 imbalance_ratio=3.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=36 tokens_per_ep_rank=[32768, 33096, 32750, 33096, 32767, 32787, 32625, 32255] avg=32768.0000 max=33096 max_rank=1 min=32255 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65342 max_load_reduction_pct=0.663788 moved_to_replicas=84880 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=37 tokens_per_ep_rank=[17, 32793, 50, 32752, 50, 32743, 65485, 98254] avg=32768.0000 max=98254 max_rank=7 min=17 min_rank=0 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=37 tokens_per_ep_rank=[32759, 32793, 32466, 32752, 32439, 32743, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32439 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=97547 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 65508, 32, 32818, 0, 32785, 65490] avg=32768.0000 max=65508 max_rank=2 min=0 min_rank=5 imbalance_ratio=1.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=38 tokens_per_ep_rank=[32767, 32744, 33096, 32426, 32818, 32412, 32785, 33096] avg=32768.0000 max=33096 max_rank=2 min=32412 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32412 max_load_reduction_pct=0.494779 moved_to_replicas=64806 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=39 tokens_per_ep_rank=[0, 31191, 0, 32793, 50, 163700, 33287, 1123] avg=32768.0000 max=163700 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.995728 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=39 tokens_per_ep_rank=[32742, 31788, 32742, 32793, 32791, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=130604 max_load_reduction_pct=0.797825 moved_to_replicas=130795 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=40 tokens_per_ep_rank=[45, 32857, 32742, 32793, 32693, 32718, 98273, 23] avg=32768.0000 max=98273 max_rank=6 min=23 min_rank=7 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=40 tokens_per_ep_rank=[32480, 32857, 32742, 32793, 32693, 32718, 33096, 32765] avg=32768.0000 max=33096 max_rank=6 min=32480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=65177 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=41 tokens_per_ep_rank=[32787, 25, 97868, 0, 65482, 33158, 81, 32743] avg=32768.0000 max=97868 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.986694 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=41 tokens_per_ep_rank=[32787, 32117, 33096, 32742, 33096, 33096, 32467, 32743] avg=32768.0000 max=33096 max_rank=2 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64772 max_load_reduction_pct=0.661830 moved_to_replicas=97220 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 65536, 32588, 25, 32768] avg=32768.0000 max=65536 max_rank=4 min=25 min_rank=6 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=42 tokens_per_ep_rank=[32792, 32743, 32746, 32946, 33096, 32588, 32465, 32768] avg=32768.0000 max=33096 max_rank=4 min=32465 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=32440 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=43 tokens_per_ep_rank=[4451, 59474, 43, 50, 32793, 65555, 32768, 67010] avg=32768.0000 max=67010 max_rank=7 min=43 min_rank=2 imbalance_ratio=2.044983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=43 tokens_per_ep_rank=[31414, 33097, 32784, 33096, 32793, 33096, 32768, 33096] avg=32768.0000 max=33097 max_rank=1 min=31414 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33913 max_load_reduction_pct=0.506089 moved_to_replicas=92751 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=44 tokens_per_ep_rank=[32768, 50, 32743, 32666, 25, 33981, 36897, 93014] avg=32768.0000 max=93014 max_rank=7 min=25 min_rank=4 imbalance_ratio=2.838562 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=44 tokens_per_ep_rank=[32768, 31911, 32743, 32666, 32767, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31911 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=59917 max_load_reduction_pct=0.644172 moved_to_replicas=64604 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=45 tokens_per_ep_rank=[32771, 64534, 32752, 972, 65511, 65499, 50, 55] avg=32768.0000 max=65511 max_rank=4 min=50 min_rank=6 imbalance_ratio=1.999237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=45 tokens_per_ep_rank=[32771, 33096, 32752, 32410, 33096, 33096, 32465, 32458] avg=32768.0000 max=33096 max_rank=1 min=32410 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=96256 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=46 tokens_per_ep_rank=[32728, 50, 32768, 19, 25, 32774, 98215, 65565] avg=32768.0000 max=98215 max_rank=6 min=19 min_rank=3 imbalance_ratio=2.997284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=46 tokens_per_ep_rank=[32728, 32519, 32768, 32761, 32402, 32774, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65119 max_load_reduction_pct=0.663025 moved_to_replicas=97588 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=47 tokens_per_ep_rank=[61, 54, 65497, 65502, 32747, 65527, 20, 32736] avg=32768.0000 max=65527 max_rank=5 min=20 min_rank=6 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=47 tokens_per_ep_rank=[32462, 32460, 33096, 33096, 32747, 33096, 32451, 32736] avg=32768.0000 max=33096 max_rank=2 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=97238 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=48 tokens_per_ep_rank=[32667, 65467, 32777, 32768, 50, 32839, 65481, 95] avg=32768.0000 max=65481 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.998322 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=48 tokens_per_ep_rank=[32667, 33096, 32777, 32768, 32435, 32839, 33096, 32466] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=64756 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=48 total_expected_recv=1573668 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374016 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=48 total_expected_recv=1570049 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378496 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=48 total_expected_recv=1575100 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373248 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=48 total_expected_recv=1571037 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378624 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=48 total_expected_recv=1574309 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374400 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=48 total_expected_recv=1574378 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374272 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=48 total_expected_recv=1569070 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377600 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=48 total_expected_recv=1575301 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374144 peak_demand=33096 required_capacity=1.010010 - [2026-08-26 15:01:24] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 735.9 | throughput per GPU (TFLOP/s/GPU): 121.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.007059E+00 | loss scale: 1.0 | grad norm: 3.702 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=30930 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=30930 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[25268, 18240, 40342, 37127, 37934, 30426, 35663, 37144] avg=32768.0000 max=40342 max_rank=2 min=18240 min_rank=1 imbalance_ratio=1.231140 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[25268, 18240, 40342, 37127, 37934, 30426, 35663, 37144] avg=32768.0000 max=40342 max_rank=2 min=18240 min_rank=1 imbalance_ratio=1.231140 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[25268, 18240, 40342, 37127, 37934, 30426, 35663, 37144] avg=32768.0000 max=40342 max_rank=2 min=18240 min_rank=1 imbalance_ratio=1.231140 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[25268, 18240, 40342, 37127, 37934, 30426, 35663, 37144] avg=32768.0000 max=40342 max_rank=2 min=18240 min_rank=1 imbalance_ratio=1.231140 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30575 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=7245 max_load_reduction_pct=0.179590 moved_to_replicas=22730 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[25268, 18240, 40342, 37127, 37934, 30426, 35663, 37144] avg=32768.0000 max=40342 max_rank=2 min=18240 min_rank=1 imbalance_ratio=1.231140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[25268, 18240, 40342, 37127, 37934, 30426, 35663, 37144] avg=32768.0000 max=40342 max_rank=2 min=18240 min_rank=1 imbalance_ratio=1.231140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30575 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=7245 max_load_reduction_pct=0.179590 moved_to_replicas=22730 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30575 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=7245 max_load_reduction_pct=0.179590 moved_to_replicas=22730 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30575 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=7245 max_load_reduction_pct=0.179590 moved_to_replicas=22730 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30575 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=7245 max_load_reduction_pct=0.179590 moved_to_replicas=22730 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[1164, 95437, 10199, 2019, 140, 2796, 65835, 84554] avg=32768.0000 max=95437 max_rank=1 min=140 min_rank=4 imbalance_ratio=2.912506 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30575 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=7245 max_load_reduction_pct=0.179590 moved_to_replicas=22730 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[25268, 18240, 40342, 37127, 37934, 30426, 35663, 37144] avg=32768.0000 max=40342 max_rank=2 min=18240 min_rank=1 imbalance_ratio=1.231140 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[1164, 95437, 10199, 2019, 140, 2796, 65835, 84554] avg=32768.0000 max=95437 max_rank=1 min=140 min_rank=4 imbalance_ratio=2.912506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[1164, 95437, 10199, 2019, 140, 2796, 65835, 84554] avg=32768.0000 max=95437 max_rank=1 min=140 min_rank=4 imbalance_ratio=2.912506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[1164, 95437, 10199, 2019, 140, 2796, 65835, 84554] avg=32768.0000 max=95437 max_rank=1 min=140 min_rank=4 imbalance_ratio=2.912506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] avg=32768.0000 max=33096 max_rank=1 min=31165 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62341 max_load_reduction_pct=0.653216 moved_to_replicas=146538 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[1164, 95437, 10199, 2019, 140, 2796, 65835, 84554] avg=32768.0000 max=95437 max_rank=1 min=140 min_rank=4 imbalance_ratio=2.912506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30575 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=7245 max_load_reduction_pct=0.179590 moved_to_replicas=22730 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] avg=32768.0000 max=33096 max_rank=1 min=31165 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62341 max_load_reduction_pct=0.653216 moved_to_replicas=146538 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[1164, 95437, 10199, 2019, 140, 2796, 65835, 84554] avg=32768.0000 max=95437 max_rank=1 min=140 min_rank=4 imbalance_ratio=2.912506 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] avg=32768.0000 max=33096 max_rank=1 min=31165 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62341 max_load_reduction_pct=0.653216 moved_to_replicas=146538 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] avg=32768.0000 max=33096 max_rank=1 min=31165 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62341 max_load_reduction_pct=0.653216 moved_to_replicas=146538 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] avg=32768.0000 max=33096 max_rank=1 min=31165 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62341 max_load_reduction_pct=0.653216 moved_to_replicas=146538 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32790, 24391, 35128, 91, 75627, 32697, 29556, 31864] avg=32768.0000 max=75627 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.307953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] avg=32768.0000 max=33096 max_rank=1 min=31165 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62341 max_load_reduction_pct=0.653216 moved_to_replicas=146538 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[1164, 95437, 10199, 2019, 140, 2796, 65835, 84554] avg=32768.0000 max=95437 max_rank=1 min=140 min_rank=4 imbalance_ratio=2.912506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32790, 24391, 35128, 91, 75627, 32697, 29556, 31864] avg=32768.0000 max=75627 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.307953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32790, 24391, 35128, 91, 75627, 32697, 29556, 31864] avg=32768.0000 max=75627 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.307953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32790, 24391, 35128, 91, 75627, 32697, 29556, 31864] avg=32768.0000 max=75627 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.307953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] avg=32768.0000 max=33096 max_rank=1 min=31864 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42531 max_load_reduction_pct=0.562379 moved_to_replicas=44563 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] avg=32768.0000 max=33096 max_rank=1 min=31165 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62341 max_load_reduction_pct=0.653216 moved_to_replicas=146538 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] avg=32768.0000 max=33096 max_rank=1 min=31864 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42531 max_load_reduction_pct=0.562379 moved_to_replicas=44563 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32790, 24391, 35128, 91, 75627, 32697, 29556, 31864] avg=32768.0000 max=75627 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.307953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[25268, 18240, 40342, 37127, 37934, 30426, 35663, 37144] avg=32768.0000 max=40342 max_rank=2 min=18240 min_rank=1 imbalance_ratio=1.231140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32790, 24391, 35128, 91, 75627, 32697, 29556, 31864] avg=32768.0000 max=75627 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.307953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] avg=32768.0000 max=33096 max_rank=1 min=31864 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42531 max_load_reduction_pct=0.562379 moved_to_replicas=44563 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] avg=32768.0000 max=33096 max_rank=1 min=31864 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42531 max_load_reduction_pct=0.562379 moved_to_replicas=44563 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[3308, 31838, 31463, 32945, 32754, 31627, 32789, 65420] avg=32768.0000 max=65420 max_rank=7 min=3308 min_rank=0 imbalance_ratio=1.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] avg=32768.0000 max=33096 max_rank=1 min=31864 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42531 max_load_reduction_pct=0.562379 moved_to_replicas=44563 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] avg=32768.0000 max=33096 max_rank=1 min=31864 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42531 max_load_reduction_pct=0.562379 moved_to_replicas=44563 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[33096, 30575, 33096, 33096, 33096, 32992, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30575 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=7245 max_load_reduction_pct=0.179590 moved_to_replicas=22730 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[3308, 31838, 31463, 32945, 32754, 31627, 32789, 65420] avg=32768.0000 max=65420 max_rank=7 min=3308 min_rank=0 imbalance_ratio=1.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32790, 24391, 35128, 91, 75627, 32697, 29556, 31864] avg=32768.0000 max=75627 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.307953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[3308, 31838, 31463, 32945, 32754, 31627, 32789, 65420] avg=32768.0000 max=65420 max_rank=7 min=3308 min_rank=0 imbalance_ratio=1.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] avg=32768.0000 max=33096 max_rank=0 min=31838 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32324 max_load_reduction_pct=0.494100 moved_to_replicas=32324 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[3308, 31838, 31463, 32945, 32754, 31627, 32789, 65420] avg=32768.0000 max=65420 max_rank=7 min=3308 min_rank=0 imbalance_ratio=1.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] avg=32768.0000 max=33096 max_rank=0 min=31838 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32324 max_load_reduction_pct=0.494100 moved_to_replicas=32324 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[3308, 31838, 31463, 32945, 32754, 31627, 32789, 65420] avg=32768.0000 max=65420 max_rank=7 min=3308 min_rank=0 imbalance_ratio=1.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] avg=32768.0000 max=33096 max_rank=1 min=31864 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42531 max_load_reduction_pct=0.562379 moved_to_replicas=44563 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[3308, 31838, 31463, 32945, 32754, 31627, 32789, 65420] avg=32768.0000 max=65420 max_rank=7 min=3308 min_rank=0 imbalance_ratio=1.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] avg=32768.0000 max=33096 max_rank=0 min=31838 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32324 max_load_reduction_pct=0.494100 moved_to_replicas=32324 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[1164, 95437, 10199, 2019, 140, 2796, 65835, 84554] avg=32768.0000 max=95437 max_rank=1 min=140 min_rank=4 imbalance_ratio=2.912506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] avg=32768.0000 max=33096 max_rank=0 min=31838 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32324 max_load_reduction_pct=0.494100 moved_to_replicas=32324 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49501, 33584, 183, 36068, 32476, 32800, 77249, 283] avg=32768.0000 max=77249 max_rank=6 min=183 min_rank=2 imbalance_ratio=2.357452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] avg=32768.0000 max=33096 max_rank=0 min=31838 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32324 max_load_reduction_pct=0.494100 moved_to_replicas=32324 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] avg=32768.0000 max=33096 max_rank=0 min=31838 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32324 max_load_reduction_pct=0.494100 moved_to_replicas=32324 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49501, 33584, 183, 36068, 32476, 32800, 77249, 283] avg=32768.0000 max=77249 max_rank=6 min=183 min_rank=2 imbalance_ratio=2.357452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[31165, 33096, 33095, 33096, 32869, 32632, 33096, 33095] avg=32768.0000 max=33096 max_rank=1 min=31165 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=62341 max_load_reduction_pct=0.653216 moved_to_replicas=146538 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[3308, 31838, 31463, 32945, 32754, 31627, 32789, 65420] avg=32768.0000 max=65420 max_rank=7 min=3308 min_rank=0 imbalance_ratio=1.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] avg=32768.0000 max=33096 max_rank=0 min=31555 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44153 max_load_reduction_pct=0.571567 moved_to_replicas=64018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49501, 33584, 183, 36068, 32476, 32800, 77249, 283] avg=32768.0000 max=77249 max_rank=6 min=183 min_rank=2 imbalance_ratio=2.357452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49501, 33584, 183, 36068, 32476, 32800, 77249, 283] avg=32768.0000 max=77249 max_rank=6 min=183 min_rank=2 imbalance_ratio=2.357452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] avg=32768.0000 max=33096 max_rank=0 min=31555 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44153 max_load_reduction_pct=0.571567 moved_to_replicas=64018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49501, 33584, 183, 36068, 32476, 32800, 77249, 283] avg=32768.0000 max=77249 max_rank=6 min=183 min_rank=2 imbalance_ratio=2.357452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] avg=32768.0000 max=33096 max_rank=0 min=31838 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32324 max_load_reduction_pct=0.494100 moved_to_replicas=32324 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49501, 33584, 183, 36068, 32476, 32800, 77249, 283] avg=32768.0000 max=77249 max_rank=6 min=183 min_rank=2 imbalance_ratio=2.357452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] avg=32768.0000 max=33096 max_rank=0 min=31555 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44153 max_load_reduction_pct=0.571567 moved_to_replicas=64018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32790, 24391, 35128, 91, 75627, 32697, 29556, 31864] avg=32768.0000 max=75627 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.307953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] avg=32768.0000 max=33096 max_rank=0 min=31555 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44153 max_load_reduction_pct=0.571567 moved_to_replicas=64018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[34242, 3, 64005, 82, 32770, 32789, 42, 98211] avg=32768.0000 max=98211 max_rank=7 min=3 min_rank=1 imbalance_ratio=2.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] avg=32768.0000 max=33096 max_rank=0 min=31555 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44153 max_load_reduction_pct=0.571567 moved_to_replicas=64018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] avg=32768.0000 max=33096 max_rank=0 min=31555 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44153 max_load_reduction_pct=0.571567 moved_to_replicas=64018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[34242, 3, 64005, 82, 32770, 32789, 42, 98211] avg=32768.0000 max=98211 max_rank=7 min=3 min_rank=1 imbalance_ratio=2.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32790, 33096, 33096, 32813, 33096, 32697, 32692, 31864] avg=32768.0000 max=33096 max_rank=1 min=31864 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42531 max_load_reduction_pct=0.562379 moved_to_replicas=44563 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49501, 33584, 183, 36068, 32476, 32800, 77249, 283] avg=32768.0000 max=77249 max_rank=6 min=183 min_rank=2 imbalance_ratio=2.357452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[34242, 3, 64005, 82, 32770, 32789, 42, 98211] avg=32768.0000 max=98211 max_rank=7 min=3 min_rank=1 imbalance_ratio=2.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] avg=32768.0000 max=33096 max_rank=0 min=32137 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65115 max_load_reduction_pct=0.663011 moved_to_replicas=97170 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[34242, 3, 64005, 82, 32770, 32789, 42, 98211] avg=32768.0000 max=98211 max_rank=7 min=3 min_rank=1 imbalance_ratio=2.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] avg=32768.0000 max=33096 max_rank=0 min=32137 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65115 max_load_reduction_pct=0.663011 moved_to_replicas=97170 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[34242, 3, 64005, 82, 32770, 32789, 42, 98211] avg=32768.0000 max=98211 max_rank=7 min=3 min_rank=1 imbalance_ratio=2.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] avg=32768.0000 max=33096 max_rank=0 min=31555 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44153 max_load_reduction_pct=0.571567 moved_to_replicas=64018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[34242, 3, 64005, 82, 32770, 32789, 42, 98211] avg=32768.0000 max=98211 max_rank=7 min=3 min_rank=1 imbalance_ratio=2.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] avg=32768.0000 max=33096 max_rank=0 min=32137 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65115 max_load_reduction_pct=0.663011 moved_to_replicas=97170 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] avg=32768.0000 max=33096 max_rank=0 min=32137 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65115 max_load_reduction_pct=0.663011 moved_to_replicas=97170 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[3308, 31838, 31463, 32945, 32754, 31627, 32789, 65420] avg=32768.0000 max=65420 max_rank=7 min=3308 min_rank=0 imbalance_ratio=1.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[74620, 25, 54, 32283, 24218, 65539, 32636, 32769] avg=32768.0000 max=74620 max_rank=0 min=25 min_rank=1 imbalance_ratio=2.277222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] avg=32768.0000 max=33096 max_rank=0 min=32137 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65115 max_load_reduction_pct=0.663011 moved_to_replicas=97170 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[74620, 25, 54, 32283, 24218, 65539, 32636, 32769] avg=32768.0000 max=74620 max_rank=0 min=25 min_rank=1 imbalance_ratio=2.277222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] avg=32768.0000 max=33096 max_rank=0 min=32137 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65115 max_load_reduction_pct=0.663011 moved_to_replicas=97170 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[34242, 3, 64005, 82, 32770, 32789, 42, 98211] avg=32768.0000 max=98211 max_rank=7 min=3 min_rank=1 imbalance_ratio=2.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[33096, 31838, 33096, 32945, 32754, 32530, 32789, 33096] avg=32768.0000 max=33096 max_rank=0 min=31838 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32324 max_load_reduction_pct=0.494100 moved_to_replicas=32324 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[74620, 25, 54, 32283, 24218, 65539, 32636, 32769] avg=32768.0000 max=74620 max_rank=0 min=25 min_rank=1 imbalance_ratio=2.277222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[74620, 25, 54, 32283, 24218, 65539, 32636, 32769] avg=32768.0000 max=74620 max_rank=0 min=25 min_rank=1 imbalance_ratio=2.277222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] avg=32768.0000 max=33097 max_rank=2 min=32283 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41523 max_load_reduction_pct=0.556459 moved_to_replicas=73967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] avg=32768.0000 max=33097 max_rank=2 min=32283 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41523 max_load_reduction_pct=0.556459 moved_to_replicas=73967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[74620, 25, 54, 32283, 24218, 65539, 32636, 32769] avg=32768.0000 max=74620 max_rank=0 min=25 min_rank=1 imbalance_ratio=2.277222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] avg=32768.0000 max=33096 max_rank=0 min=32137 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65115 max_load_reduction_pct=0.663011 moved_to_replicas=97170 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] avg=32768.0000 max=33097 max_rank=2 min=32283 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41523 max_load_reduction_pct=0.556459 moved_to_replicas=73967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[74620, 25, 54, 32283, 24218, 65539, 32636, 32769] avg=32768.0000 max=74620 max_rank=0 min=25 min_rank=1 imbalance_ratio=2.277222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] avg=32768.0000 max=33097 max_rank=2 min=32283 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41523 max_load_reduction_pct=0.556459 moved_to_replicas=73967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49501, 33584, 183, 36068, 32476, 32800, 77249, 283] avg=32768.0000 max=77249 max_rank=6 min=183 min_rank=2 imbalance_ratio=2.357452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[1179, 31850, 181, 32364, 98284, 32870, 2, 65414] avg=32768.0000 max=98284 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] avg=32768.0000 max=33097 max_rank=2 min=32283 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41523 max_load_reduction_pct=0.556459 moved_to_replicas=73967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[1179, 31850, 181, 32364, 98284, 32870, 2, 65414] avg=32768.0000 max=98284 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] avg=32768.0000 max=33097 max_rank=2 min=32283 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41523 max_load_reduction_pct=0.556459 moved_to_replicas=73967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[74620, 25, 54, 32283, 24218, 65539, 32636, 32769] avg=32768.0000 max=74620 max_rank=0 min=25 min_rank=1 imbalance_ratio=2.277222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[1179, 31850, 181, 32364, 98284, 32870, 2, 65414] avg=32768.0000 max=98284 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[1179, 31850, 181, 32364, 98284, 32870, 2, 65414] avg=32768.0000 max=98284 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[33096, 33096, 32929, 33096, 32476, 32800, 33096, 31555] avg=32768.0000 max=33096 max_rank=0 min=31555 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44153 max_load_reduction_pct=0.571567 moved_to_replicas=64018 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] avg=32768.0000 max=33096 max_rank=0 min=32251 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=97506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] avg=32768.0000 max=33096 max_rank=0 min=32251 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=97506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] avg=32768.0000 max=33097 max_rank=2 min=32283 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41523 max_load_reduction_pct=0.556459 moved_to_replicas=73967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] avg=32768.0000 max=33096 max_rank=0 min=32251 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=97506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[1179, 31850, 181, 32364, 98284, 32870, 2, 65414] avg=32768.0000 max=98284 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] avg=32768.0000 max=33096 max_rank=0 min=32251 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=97506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[1179, 31850, 181, 32364, 98284, 32870, 2, 65414] avg=32768.0000 max=98284 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[41751, 32740, 12187, 21, 77147, 42, 32738, 65518] avg=32768.0000 max=77147 max_rank=4 min=21 min_rank=3 imbalance_ratio=2.354340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[34242, 3, 64005, 82, 32770, 32789, 42, 98211] avg=32768.0000 max=98211 max_rank=7 min=3 min_rank=1 imbalance_ratio=2.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[41751, 32740, 12187, 21, 77147, 42, 32738, 65518] avg=32768.0000 max=77147 max_rank=4 min=21 min_rank=3 imbalance_ratio=2.354340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] avg=32768.0000 max=33096 max_rank=0 min=32251 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=97506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] avg=32768.0000 max=33096 max_rank=0 min=32251 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=97506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[1179, 31850, 181, 32364, 98284, 32870, 2, 65414] avg=32768.0000 max=98284 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[41751, 32740, 12187, 21, 77147, 42, 32738, 65518] avg=32768.0000 max=77147 max_rank=4 min=21 min_rank=3 imbalance_ratio=2.354340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[41751, 32740, 12187, 21, 77147, 42, 32738, 65518] avg=32768.0000 max=77147 max_rank=4 min=21 min_rank=3 imbalance_ratio=2.354340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] avg=32768.0000 max=33096 max_rank=0 min=31515 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=44051 max_load_reduction_pct=0.571001 moved_to_replicas=85128 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32747, 33096, 32137, 32770, 32789, 32413, 33096] avg=32768.0000 max=33096 max_rank=0 min=32137 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65115 max_load_reduction_pct=0.663011 moved_to_replicas=97170 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] avg=32768.0000 max=33096 max_rank=0 min=31515 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=44051 max_load_reduction_pct=0.571001 moved_to_replicas=85128 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] avg=32768.0000 max=33096 max_rank=0 min=32251 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=97506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] avg=32768.0000 max=33096 max_rank=0 min=31515 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=44051 max_load_reduction_pct=0.571001 moved_to_replicas=85128 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[41751, 32740, 12187, 21, 77147, 42, 32738, 65518] avg=32768.0000 max=77147 max_rank=4 min=21 min_rank=3 imbalance_ratio=2.354340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] avg=32768.0000 max=33096 max_rank=0 min=31515 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=44051 max_load_reduction_pct=0.571001 moved_to_replicas=85128 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[41751, 32740, 12187, 21, 77147, 42, 32738, 65518] avg=32768.0000 max=77147 max_rank=4 min=21 min_rank=3 imbalance_ratio=2.354340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[64484, 63154, 25, 32770, 22, 36067, 32769, 32853] avg=32768.0000 max=64484 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.967896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[64484, 63154, 25, 32770, 22, 36067, 32769, 32853] avg=32768.0000 max=64484 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.967896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] avg=32768.0000 max=33096 max_rank=0 min=31515 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=44051 max_load_reduction_pct=0.571001 moved_to_replicas=85128 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[74620, 25, 54, 32283, 24218, 65539, 32636, 32769] avg=32768.0000 max=74620 max_rank=0 min=25 min_rank=1 imbalance_ratio=2.277222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] avg=32768.0000 max=33096 max_rank=0 min=31515 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=44051 max_load_reduction_pct=0.571001 moved_to_replicas=85128 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[41751, 32740, 12187, 21, 77147, 42, 32738, 65518] avg=32768.0000 max=77147 max_rank=4 min=21 min_rank=3 imbalance_ratio=2.354340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[64484, 63154, 25, 32770, 22, 36067, 32769, 32853] avg=32768.0000 max=64484 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.967896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[64484, 63154, 25, 32770, 22, 36067, 32769, 32853] avg=32768.0000 max=64484 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.967896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] avg=32768.0000 max=33096 max_rank=0 min=31410 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31388 max_load_reduction_pct=0.486756 moved_to_replicas=64417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] avg=32768.0000 max=33096 max_rank=0 min=31410 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31388 max_load_reduction_pct=0.486756 moved_to_replicas=64417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[33096, 32760, 33097, 32283, 32407, 33096, 32636, 32769] avg=32768.0000 max=33097 max_rank=2 min=32283 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41523 max_load_reduction_pct=0.556459 moved_to_replicas=73967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] avg=32768.0000 max=33096 max_rank=0 min=31515 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=44051 max_load_reduction_pct=0.571001 moved_to_replicas=85128 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] avg=32768.0000 max=33096 max_rank=0 min=31410 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31388 max_load_reduction_pct=0.486756 moved_to_replicas=64417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[64484, 63154, 25, 32770, 22, 36067, 32769, 32853] avg=32768.0000 max=64484 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.967896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] avg=32768.0000 max=33096 max_rank=0 min=31410 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31388 max_load_reduction_pct=0.486756 moved_to_replicas=64417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[64484, 63154, 25, 32770, 22, 36067, 32769, 32853] avg=32768.0000 max=64484 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.967896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[21, 32761, 110, 33544, 33086, 98163, 0, 64459] avg=32768.0000 max=98163 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[21, 32761, 110, 33544, 33086, 98163, 0, 64459] avg=32768.0000 max=98163 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] avg=32768.0000 max=33096 max_rank=0 min=31410 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31388 max_load_reduction_pct=0.486756 moved_to_replicas=64417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[1179, 31850, 181, 32364, 98284, 32870, 2, 65414] avg=32768.0000 max=98284 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] avg=32768.0000 max=33096 max_rank=0 min=31410 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31388 max_load_reduction_pct=0.486756 moved_to_replicas=64417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[64484, 63154, 25, 32770, 22, 36067, 32769, 32853] avg=32768.0000 max=64484 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.967896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[21, 32761, 110, 33544, 33086, 98163, 0, 64459] avg=32768.0000 max=98163 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[21, 32761, 110, 33544, 33086, 98163, 0, 64459] avg=32768.0000 max=98163 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] avg=32768.0000 max=33096 max_rank=3 min=31921 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65067 max_load_reduction_pct=0.662846 moved_to_replicas=96878 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] avg=32768.0000 max=33096 max_rank=3 min=31921 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65067 max_load_reduction_pct=0.662846 moved_to_replicas=96878 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[33096, 32251, 32623, 32364, 33096, 32870, 32748, 33096] avg=32768.0000 max=33096 max_rank=0 min=32251 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=97506 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] avg=32768.0000 max=33096 max_rank=0 min=31410 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31388 max_load_reduction_pct=0.486756 moved_to_replicas=64417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] avg=32768.0000 max=33096 max_rank=3 min=31921 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65067 max_load_reduction_pct=0.662846 moved_to_replicas=96878 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] avg=32768.0000 max=33096 max_rank=3 min=31921 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65067 max_load_reduction_pct=0.662846 moved_to_replicas=96878 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[21, 32761, 110, 33544, 33086, 98163, 0, 64459] avg=32768.0000 max=98163 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[21, 32761, 110, 33544, 33086, 98163, 0, 64459] avg=32768.0000 max=98163 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[65367, 1016, 49, 18, 32778, 32700, 64697, 65519] avg=32768.0000 max=65519 max_rank=7 min=18 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[65367, 1016, 49, 18, 32778, 32700, 64697, 65519] avg=32768.0000 max=65519 max_rank=7 min=18 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] avg=32768.0000 max=33096 max_rank=3 min=31921 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65067 max_load_reduction_pct=0.662846 moved_to_replicas=96878 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[41751, 32740, 12187, 21, 77147, 42, 32738, 65518] avg=32768.0000 max=77147 max_rank=4 min=21 min_rank=3 imbalance_ratio=2.354340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] avg=32768.0000 max=33096 max_rank=3 min=31921 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65067 max_load_reduction_pct=0.662846 moved_to_replicas=96878 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[21, 32761, 110, 33544, 33086, 98163, 0, 64459] avg=32768.0000 max=98163 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[65367, 1016, 49, 18, 32778, 32700, 64697, 65519] avg=32768.0000 max=65519 max_rank=7 min=18 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[65367, 1016, 49, 18, 32778, 32700, 64697, 65519] avg=32768.0000 max=65519 max_rank=7 min=18 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=96295 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=96295 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[33096, 32740, 31515, 32767, 33096, 33096, 32738, 33096] avg=32768.0000 max=33096 max_rank=0 min=31515 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=44051 max_load_reduction_pct=0.571001 moved_to_replicas=85128 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] avg=32768.0000 max=33096 max_rank=3 min=31921 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65067 max_load_reduction_pct=0.662846 moved_to_replicas=96878 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=96295 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=96295 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[65367, 1016, 49, 18, 32778, 32700, 64697, 65519] avg=32768.0000 max=65519 max_rank=7 min=18 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[65367, 1016, 49, 18, 32778, 32700, 64697, 65519] avg=32768.0000 max=65519 max_rank=7 min=18 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32767, 65503, 32750, 65493, 45, 23, 47, 65516] avg=32768.0000 max=65516 max_rank=7 min=23 min_rank=5 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32767, 65503, 32750, 65493, 45, 23, 47, 65516] avg=32768.0000 max=65516 max_rank=7 min=23 min_rank=5 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=96295 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[65367, 1016, 49, 18, 32778, 32700, 64697, 65519] avg=32768.0000 max=65519 max_rank=7 min=18 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=96295 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[64484, 63154, 25, 32770, 22, 36067, 32769, 32853] avg=32768.0000 max=64484 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.967896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32767, 65503, 32750, 65493, 45, 23, 47, 65516] avg=32768.0000 max=65516 max_rank=7 min=23 min_rank=5 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32767, 65503, 32750, 65493, 45, 23, 47, 65516] avg=32768.0000 max=65516 max_rank=7 min=23 min_rank=5 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97224 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97224 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=96295 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97224 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32767, 65503, 32750, 65493, 45, 23, 47, 65516] avg=32768.0000 max=65516 max_rank=7 min=23 min_rank=5 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33054, 32770, 31410, 33096, 32769, 32853] avg=32768.0000 max=33096 max_rank=0 min=31410 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31388 max_load_reduction_pct=0.486756 moved_to_replicas=64417 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97224 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32767, 65503, 32750, 65493, 45, 23, 47, 65516] avg=32768.0000 max=65516 max_rank=7 min=23 min_rank=5 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[1, 32917, 32046, 74289, 32757, 32740, 57379, 15] avg=32768.0000 max=74289 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.267120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[1, 32917, 32046, 74289, 32757, 32740, 57379, 15] avg=32768.0000 max=74289 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.267120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97224 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32767, 65503, 32750, 65493, 45, 23, 47, 65516] avg=32768.0000 max=65516 max_rank=7 min=23 min_rank=5 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97224 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[1, 32917, 32046, 74289, 32757, 32740, 57379, 15] avg=32768.0000 max=74289 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.267120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] avg=32768.0000 max=33096 max_rank=3 min=32046 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41193 max_load_reduction_pct=0.554497 moved_to_replicas=65476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] avg=32768.0000 max=33096 max_rank=3 min=32046 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41193 max_load_reduction_pct=0.554497 moved_to_replicas=65476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[1, 32917, 32046, 74289, 32757, 32740, 57379, 15] avg=32768.0000 max=74289 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.267120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[21, 32761, 110, 33544, 33086, 98163, 0, 64459] avg=32768.0000 max=98163 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.995697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97224 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] avg=32768.0000 max=33096 max_rank=3 min=32046 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41193 max_load_reduction_pct=0.554497 moved_to_replicas=65476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] avg=32768.0000 max=33096 max_rank=3 min=32046 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41193 max_load_reduction_pct=0.554497 moved_to_replicas=65476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[1, 32917, 32046, 74289, 32757, 32740, 57379, 15] avg=32768.0000 max=74289 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.267120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32346, 32761, 31921, 33096, 33086, 33096, 32742, 33096] avg=32768.0000 max=33096 max_rank=3 min=31921 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65067 max_load_reduction_pct=0.662846 moved_to_replicas=96878 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[1, 32917, 32046, 74289, 32757, 32740, 57379, 15] avg=32768.0000 max=74289 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.267120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32791, 96961, 21, 32790, 2, 1593, 65218, 32768] avg=32768.0000 max=96961 max_rank=1 min=2 min_rank=4 imbalance_ratio=2.959015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32791, 96961, 21, 32790, 2, 1593, 65218, 32768] avg=32768.0000 max=96961 max_rank=1 min=2 min_rank=4 imbalance_ratio=2.959015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[1, 32917, 32046, 74289, 32757, 32740, 57379, 15] avg=32768.0000 max=74289 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.267120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] avg=32768.0000 max=33096 max_rank=3 min=32046 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41193 max_load_reduction_pct=0.554497 moved_to_replicas=65476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] avg=32768.0000 max=33096 max_rank=3 min=32046 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41193 max_load_reduction_pct=0.554497 moved_to_replicas=65476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32791, 96961, 21, 32790, 2, 1593, 65218, 32768] avg=32768.0000 max=96961 max_rank=1 min=2 min_rank=4 imbalance_ratio=2.959015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31759 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63865 max_load_reduction_pct=0.658667 moved_to_replicas=95987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31759 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63865 max_load_reduction_pct=0.658667 moved_to_replicas=95987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32791, 96961, 21, 32790, 2, 1593, 65218, 32768] avg=32768.0000 max=96961 max_rank=1 min=2 min_rank=4 imbalance_ratio=2.959015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[65367, 1016, 49, 18, 32778, 32700, 64697, 65519] avg=32768.0000 max=65519 max_rank=7 min=18 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] avg=32768.0000 max=33096 max_rank=3 min=32046 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41193 max_load_reduction_pct=0.554497 moved_to_replicas=65476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31759 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63865 max_load_reduction_pct=0.658667 moved_to_replicas=95987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31759 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63865 max_load_reduction_pct=0.658667 moved_to_replicas=95987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32791, 96961, 21, 32790, 2, 1593, 65218, 32768] avg=32768.0000 max=96961 max_rank=1 min=2 min_rank=4 imbalance_ratio=2.959015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32791, 96961, 21, 32790, 2, 1593, 65218, 32768] avg=32768.0000 max=96961 max_rank=1 min=2 min_rank=4 imbalance_ratio=2.959015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[33096, 32617, 32320, 32441, 32778, 32700, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=96295 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 93, 65492] avg=32768.0000 max=65492 max_rank=7 min=93 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 93, 65492] avg=32768.0000 max=65492 max_rank=7 min=93 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32791, 96961, 21, 32790, 2, 1593, 65218, 32768] avg=32768.0000 max=96961 max_rank=1 min=2 min_rank=4 imbalance_ratio=2.959015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31759 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63865 max_load_reduction_pct=0.658667 moved_to_replicas=95987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31759 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63865 max_load_reduction_pct=0.658667 moved_to_replicas=95987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] avg=32768.0000 max=33096 max_rank=7 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=32396 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 93, 65492] avg=32768.0000 max=65492 max_rank=7 min=93 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] avg=32768.0000 max=33096 max_rank=7 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=32396 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 93, 65492] avg=32768.0000 max=65492 max_rank=7 min=93 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31759 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63865 max_load_reduction_pct=0.658667 moved_to_replicas=95987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32767, 65503, 32750, 65493, 45, 23, 47, 65516] avg=32768.0000 max=65516 max_rank=7 min=23 min_rank=5 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] avg=32768.0000 max=33096 max_rank=7 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=32396 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 93, 65492] avg=32768.0000 max=65492 max_rank=7 min=93 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] avg=32768.0000 max=33096 max_rank=7 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=32396 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 93, 65492] avg=32768.0000 max=65492 max_rank=7 min=93 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 21, 98180, 65514, 32757, 21, 21] avg=32768.0000 max=98180 max_rank=3 min=21 min_rank=2 imbalance_ratio=2.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32767, 33096, 32750, 33096, 32452, 32443, 32444, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=97224 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 21, 98180, 65514, 32757, 21, 21] avg=32768.0000 max=98180 max_rank=3 min=21 min_rank=2 imbalance_ratio=2.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 93, 65492] avg=32768.0000 max=65492 max_rank=7 min=93 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] avg=32768.0000 max=33096 max_rank=7 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=32396 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] avg=32768.0000 max=33096 max_rank=7 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=32396 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] avg=32768.0000 max=33096 max_rank=3 min=32359 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.662905 moved_to_replicas=97502 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 21, 98180, 65514, 32757, 21, 21] avg=32768.0000 max=98180 max_rank=3 min=21 min_rank=2 imbalance_ratio=2.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] avg=32768.0000 max=33096 max_rank=3 min=32359 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.662905 moved_to_replicas=97502 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 21, 98180, 65514, 32757, 21, 21] avg=32768.0000 max=98180 max_rank=3 min=21 min_rank=2 imbalance_ratio=2.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] avg=32768.0000 max=33096 max_rank=7 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=32396 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[1, 32917, 32046, 74289, 32757, 32740, 57379, 15] avg=32768.0000 max=74289 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.267120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] avg=32768.0000 max=33096 max_rank=3 min=32359 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.662905 moved_to_replicas=97502 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 21, 98180, 65514, 32757, 21, 21] avg=32768.0000 max=98180 max_rank=3 min=21 min_rank=2 imbalance_ratio=2.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] avg=32768.0000 max=33096 max_rank=3 min=32359 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.662905 moved_to_replicas=97502 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[31136, 31881, 35605, 32767, 42, 65515, 32711, 32487] avg=32768.0000 max=65515 max_rank=5 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 21, 98180, 65514, 32757, 21, 21] avg=32768.0000 max=98180 max_rank=3 min=21 min_rank=2 imbalance_ratio=2.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[31136, 31881, 35605, 32767, 42, 65515, 32711, 32487] avg=32768.0000 max=65515 max_rank=5 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32747, 32917, 32046, 33096, 32757, 32740, 33096, 32745] avg=32768.0000 max=33096 max_rank=3 min=32046 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41193 max_load_reduction_pct=0.554497 moved_to_replicas=65476 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 21, 98180, 65514, 32757, 21, 21] avg=32768.0000 max=98180 max_rank=3 min=21 min_rank=2 imbalance_ratio=2.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] avg=32768.0000 max=33096 max_rank=3 min=32359 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.662905 moved_to_replicas=97502 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=34928 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] avg=32768.0000 max=33096 max_rank=3 min=32359 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.662905 moved_to_replicas=97502 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[31136, 31881, 35605, 32767, 42, 65515, 32711, 32487] avg=32768.0000 max=65515 max_rank=5 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[31136, 31881, 35605, 32767, 42, 65515, 32711, 32487] avg=32768.0000 max=65515 max_rank=5 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=34928 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] avg=32768.0000 max=33096 max_rank=3 min=32359 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.662905 moved_to_replicas=97502 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32791, 96961, 21, 32790, 2, 1593, 65218, 32768] avg=32768.0000 max=96961 max_rank=1 min=2 min_rank=4 imbalance_ratio=2.959015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=34928 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=34928 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[31136, 31881, 35605, 32767, 42, 65515, 32711, 32487] avg=32768.0000 max=65515 max_rank=5 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 45, 32790, 65485, 258, 32549, 65495] avg=32768.0000 max=65495 max_rank=7 min=45 min_rank=2 imbalance_ratio=1.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[31136, 31881, 35605, 32767, 42, 65515, 32711, 32487] avg=32768.0000 max=65515 max_rank=5 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[31136, 31881, 35605, 32767, 42, 65515, 32711, 32487] avg=32768.0000 max=65515 max_rank=5 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 45, 32790, 65485, 258, 32549, 65495] avg=32768.0000 max=65495 max_rank=7 min=45 min_rank=2 imbalance_ratio=1.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32791, 33096, 31759, 32790, 32748, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31759 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63865 max_load_reduction_pct=0.658667 moved_to_replicas=95987 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] avg=32768.0000 max=33096 max_rank=4 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32399 max_load_reduction_pct=0.494679 moved_to_replicas=64788 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=34928 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 45, 32790, 65485, 258, 32549, 65495] avg=32768.0000 max=65495 max_rank=7 min=45 min_rank=2 imbalance_ratio=1.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=34928 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=34928 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] avg=32768.0000 max=33096 max_rank=4 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32399 max_load_reduction_pct=0.494679 moved_to_replicas=64788 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 45, 32790, 65485, 258, 32549, 65495] avg=32768.0000 max=65495 max_rank=7 min=45 min_rank=2 imbalance_ratio=1.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 93, 65492] avg=32768.0000 max=65492 max_rank=7 min=93 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] avg=32768.0000 max=33096 max_rank=4 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32399 max_load_reduction_pct=0.494679 moved_to_replicas=64788 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[32789, 64586, 29, 33668, 42, 21, 98262, 32747] avg=32768.0000 max=98262 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] avg=32768.0000 max=33096 max_rank=4 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32399 max_load_reduction_pct=0.494679 moved_to_replicas=64788 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 45, 32790, 65485, 258, 32549, 65495] avg=32768.0000 max=65495 max_rank=7 min=45 min_rank=2 imbalance_ratio=1.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 45, 32790, 65485, 258, 32549, 65495] avg=32768.0000 max=65495 max_rank=7 min=45 min_rank=2 imbalance_ratio=1.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 45, 32790, 65485, 258, 32549, 65495] avg=32768.0000 max=65495 max_rank=7 min=45 min_rank=2 imbalance_ratio=1.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32776, 32577, 32747, 32769, 32872, 32818, 32489, 33096] avg=32768.0000 max=33096 max_rank=7 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=32396 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[32789, 64586, 29, 33668, 42, 21, 98262, 32747] avg=32768.0000 max=98262 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65166 max_load_reduction_pct=0.663186 moved_to_replicas=97228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] avg=32768.0000 max=33096 max_rank=4 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32399 max_load_reduction_pct=0.494679 moved_to_replicas=64788 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[32789, 64586, 29, 33668, 42, 21, 98262, 32747] avg=32768.0000 max=98262 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] avg=32768.0000 max=33096 max_rank=4 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32399 max_load_reduction_pct=0.494679 moved_to_replicas=64788 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] avg=32768.0000 max=33096 max_rank=4 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32399 max_load_reduction_pct=0.494679 moved_to_replicas=64788 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[32789, 64586, 29, 33668, 42, 21, 98262, 32747] avg=32768.0000 max=98262 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65166 max_load_reduction_pct=0.663186 moved_to_replicas=97228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 21, 98180, 65514, 32757, 21, 21] avg=32768.0000 max=98180 max_rank=3 min=21 min_rank=2 imbalance_ratio=2.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65166 max_load_reduction_pct=0.663186 moved_to_replicas=97228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33407, 65534, 34, 20, 65140] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=6 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65166 max_load_reduction_pct=0.663186 moved_to_replicas=97228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[32789, 64586, 29, 33668, 42, 21, 98262, 32747] avg=32768.0000 max=98262 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[32789, 64586, 29, 33668, 42, 21, 98262, 32747] avg=32768.0000 max=98262 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[32789, 64586, 29, 33668, 42, 21, 98262, 32747] avg=32768.0000 max=98262 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33407, 65534, 34, 20, 65140] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=6 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[32848, 32782, 32767, 33096, 33096, 32757, 32359, 32439] avg=32768.0000 max=33096 max_rank=3 min=32359 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.662905 moved_to_replicas=97502 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] avg=32768.0000 max=33096 max_rank=3 min=32389 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=64793 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65166 max_load_reduction_pct=0.663186 moved_to_replicas=97228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65166 max_load_reduction_pct=0.663186 moved_to_replicas=97228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33407, 65534, 34, 20, 65140] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=6 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33407, 65534, 34, 20, 65140] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=6 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65166 max_load_reduction_pct=0.663186 moved_to_replicas=97228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] avg=32768.0000 max=33096 max_rank=3 min=32389 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=64793 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[31136, 31881, 35605, 32767, 42, 65515, 32711, 32487] avg=32768.0000 max=65515 max_rank=5 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] avg=32768.0000 max=33096 max_rank=3 min=32389 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=64793 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[63157, 36, 9, 29057, 32762, 64840, 32765, 39518] avg=32768.0000 max=64840 max_rank=5 min=9 min_rank=2 imbalance_ratio=1.978760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] avg=32768.0000 max=33096 max_rank=3 min=32389 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=64793 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33407, 65534, 34, 20, 65140] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=6 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33407, 65534, 34, 20, 65140] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=6 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[63157, 36, 9, 29057, 32762, 64840, 32765, 39518] avg=32768.0000 max=64840 max_rank=5 min=9 min_rank=2 imbalance_ratio=1.978760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33407, 65534, 34, 20, 65140] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=6 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[33096, 32430, 33096, 32767, 32461, 33096, 32711, 32487] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=34928 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] avg=32768.0000 max=33097 max_rank=3 min=31753 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31743 max_load_reduction_pct=0.489559 moved_to_replicas=68227 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] avg=32768.0000 max=33096 max_rank=3 min=32389 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=64793 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] avg=32768.0000 max=33096 max_rank=3 min=32389 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=64793 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[63157, 36, 9, 29057, 32762, 64840, 32765, 39518] avg=32768.0000 max=64840 max_rank=5 min=9 min_rank=2 imbalance_ratio=1.978760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[63157, 36, 9, 29057, 32762, 64840, 32765, 39518] avg=32768.0000 max=64840 max_rank=5 min=9 min_rank=2 imbalance_ratio=1.978760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] avg=32768.0000 max=33097 max_rank=3 min=31753 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31743 max_load_reduction_pct=0.489559 moved_to_replicas=68227 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] avg=32768.0000 max=33096 max_rank=3 min=32389 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=64793 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[3, 65253, 65272, 42, 1253, 10, 32741, 97570] avg=32768.0000 max=97570 max_rank=7 min=3 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] avg=32768.0000 max=33097 max_rank=3 min=31753 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31743 max_load_reduction_pct=0.489559 moved_to_replicas=68227 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 45, 32790, 65485, 258, 32549, 65495] avg=32768.0000 max=65495 max_rank=7 min=45 min_rank=2 imbalance_ratio=1.998749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] avg=32768.0000 max=33097 max_rank=3 min=31753 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31743 max_load_reduction_pct=0.489559 moved_to_replicas=68227 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[63157, 36, 9, 29057, 32762, 64840, 32765, 39518] avg=32768.0000 max=64840 max_rank=5 min=9 min_rank=2 imbalance_ratio=1.978760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[63157, 36, 9, 29057, 32762, 64840, 32765, 39518] avg=32768.0000 max=64840 max_rank=5 min=9 min_rank=2 imbalance_ratio=1.978760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[3, 65253, 65272, 42, 1253, 10, 32741, 97570] avg=32768.0000 max=97570 max_rank=7 min=3 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[63157, 36, 9, 29057, 32762, 64840, 32765, 39518] avg=32768.0000 max=64840 max_rank=5 min=9 min_rank=2 imbalance_ratio=1.978760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] avg=32768.0000 max=33096 max_rank=1 min=32052 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=128807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] avg=32768.0000 max=33097 max_rank=3 min=31753 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31743 max_load_reduction_pct=0.489559 moved_to_replicas=68227 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[32776, 32746, 32444, 32790, 33096, 32647, 32549, 33096] avg=32768.0000 max=33096 max_rank=4 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32399 max_load_reduction_pct=0.494679 moved_to_replicas=64788 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] avg=32768.0000 max=33097 max_rank=3 min=31753 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31743 max_load_reduction_pct=0.489559 moved_to_replicas=68227 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] avg=32768.0000 max=33096 max_rank=1 min=32052 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=128807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[3, 65253, 65272, 42, 1253, 10, 32741, 97570] avg=32768.0000 max=97570 max_rank=7 min=3 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[3, 65253, 65272, 42, 1253, 10, 32741, 97570] avg=32768.0000 max=97570 max_rank=7 min=3 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] avg=32768.0000 max=33097 max_rank=3 min=31753 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31743 max_load_reduction_pct=0.489559 moved_to_replicas=68227 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[32768, 65450, 62, 65494, 32763, 32747, 32798, 62] avg=32768.0000 max=65494 max_rank=3 min=62 min_rank=2 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[3, 65253, 65272, 42, 1253, 10, 32741, 97570] avg=32768.0000 max=97570 max_rank=7 min=3 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] avg=32768.0000 max=33096 max_rank=1 min=32052 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=128807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] avg=32768.0000 max=33096 max_rank=1 min=32052 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=128807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[32789, 64586, 29, 33668, 42, 21, 98262, 32747] avg=32768.0000 max=98262 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[3, 65253, 65272, 42, 1253, 10, 32741, 97570] avg=32768.0000 max=97570 max_rank=7 min=3 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[32768, 65450, 62, 65494, 32763, 32747, 32798, 62] avg=32768.0000 max=65494 max_rank=3 min=62 min_rank=2 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[3, 65253, 65272, 42, 1253, 10, 32741, 97570] avg=32768.0000 max=97570 max_rank=7 min=3 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] avg=32768.0000 max=33096 max_rank=1 min=32416 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=64752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] avg=32768.0000 max=33096 max_rank=1 min=32052 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=128807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[32789, 33096, 32449, 33096, 32104, 32767, 33096, 32747] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65166 max_load_reduction_pct=0.663186 moved_to_replicas=97228 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] avg=32768.0000 max=33096 max_rank=1 min=32052 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=128807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] avg=32768.0000 max=33096 max_rank=1 min=32416 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=64752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[32768, 65450, 62, 65494, 32763, 32747, 32798, 62] avg=32768.0000 max=65494 max_rank=3 min=62 min_rank=2 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[32768, 65450, 62, 65494, 32763, 32747, 32798, 62] avg=32768.0000 max=65494 max_rank=3 min=62 min_rank=2 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] avg=32768.0000 max=33096 max_rank=1 min=32052 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=128807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=25 tokens_per_ep_rank=[4, 98283, 65494, 0, 20, 65513, 32788, 42] avg=32768.0000 max=98283 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[32768, 65450, 62, 65494, 32763, 32747, 32798, 62] avg=32768.0000 max=65494 max_rank=3 min=62 min_rank=2 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] avg=32768.0000 max=33096 max_rank=1 min=32416 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=64752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] avg=32768.0000 max=33096 max_rank=1 min=32416 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=64752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33407, 65534, 34, 20, 65140] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=6 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[32768, 65450, 62, 65494, 32763, 32747, 32798, 62] avg=32768.0000 max=65494 max_rank=3 min=62 min_rank=2 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=25 tokens_per_ep_rank=[4, 98283, 65494, 0, 20, 65513, 32788, 42] avg=32768.0000 max=98283 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=25 tokens_per_ep_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=130002 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[32768, 65450, 62, 65494, 32763, 32747, 32798, 62] avg=32768.0000 max=65494 max_rank=3 min=62 min_rank=2 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] avg=32768.0000 max=33096 max_rank=1 min=32416 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=64752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[32750, 32779, 32480, 33096, 33096, 32389, 32458, 33096] avg=32768.0000 max=33096 max_rank=3 min=32389 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=64793 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] avg=32768.0000 max=33096 max_rank=1 min=32416 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=64752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=25 tokens_per_ep_rank=[4, 98283, 65494, 0, 20, 65513, 32788, 42] avg=32768.0000 max=98283 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=25 tokens_per_ep_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=130002 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=25 tokens_per_ep_rank=[4, 98283, 65494, 0, 20, 65513, 32788, 42] avg=32768.0000 max=98283 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] avg=32768.0000 max=33096 max_rank=1 min=32416 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=64752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=26 tokens_per_ep_rank=[64, 32768, 98238, 32742, 96401, 1917, 0, 14] avg=32768.0000 max=98238 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=25 tokens_per_ep_rank=[4, 98283, 65494, 0, 20, 65513, 32788, 42] avg=32768.0000 max=98283 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=25 tokens_per_ep_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=130002 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=25 tokens_per_ep_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=130002 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[63157, 36, 9, 29057, 32762, 64840, 32765, 39518] avg=32768.0000 max=64840 max_rank=5 min=9 min_rank=2 imbalance_ratio=1.978760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=25 tokens_per_ep_rank=[4, 98283, 65494, 0, 20, 65513, 32788, 42] avg=32768.0000 max=98283 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=26 tokens_per_ep_rank=[64, 32768, 98238, 32742, 96401, 1917, 0, 14] avg=32768.0000 max=98238 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=25 tokens_per_ep_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=130002 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=26 tokens_per_ep_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=128447 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=25 tokens_per_ep_rank=[4, 98283, 65494, 0, 20, 65513, 32788, 42] avg=32768.0000 max=98283 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[33096, 32480, 31753, 33097, 32762, 33096, 32765, 33095] avg=32768.0000 max=33097 max_rank=3 min=31753 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31743 max_load_reduction_pct=0.489559 moved_to_replicas=68227 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=25 tokens_per_ep_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=130002 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=26 tokens_per_ep_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=128447 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=26 tokens_per_ep_rank=[64, 32768, 98238, 32742, 96401, 1917, 0, 14] avg=32768.0000 max=98238 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=26 tokens_per_ep_rank=[64, 32768, 98238, 32742, 96401, 1917, 0, 14] avg=32768.0000 max=98238 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=25 tokens_per_ep_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=130002 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=26 tokens_per_ep_rank=[64, 32768, 98238, 32742, 96401, 1917, 0, 14] avg=32768.0000 max=98238 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=27 tokens_per_ep_rank=[32789, 0, 1222, 65244, 131008, 67, 6, 31808] avg=32768.0000 max=131008 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=26 tokens_per_ep_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=128447 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=26 tokens_per_ep_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=128447 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[3, 65253, 65272, 42, 1253, 10, 32741, 97570] avg=32768.0000 max=97570 max_rank=7 min=3 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=26 tokens_per_ep_rank=[64, 32768, 98238, 32742, 96401, 1917, 0, 14] avg=32768.0000 max=98238 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=27 tokens_per_ep_rank=[32789, 0, 1222, 65244, 131008, 67, 6, 31808] avg=32768.0000 max=131008 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=26 tokens_per_ep_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=128447 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=27 tokens_per_ep_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] avg=32768.0000 max=33096 max_rank=2 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97912 max_load_reduction_pct=0.747374 moved_to_replicas=130060 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=26 tokens_per_ep_rank=[64, 32768, 98238, 32742, 96401, 1917, 0, 14] avg=32768.0000 max=98238 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[32749, 33096, 33096, 32218, 33096, 32052, 32741, 33096] avg=32768.0000 max=33096 max_rank=1 min=32052 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=128807 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=26 tokens_per_ep_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=128447 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=27 tokens_per_ep_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] avg=32768.0000 max=33096 max_rank=2 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97912 max_load_reduction_pct=0.747374 moved_to_replicas=130060 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=27 tokens_per_ep_rank=[32789, 0, 1222, 65244, 131008, 67, 6, 31808] avg=32768.0000 max=131008 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=27 tokens_per_ep_rank=[32789, 0, 1222, 65244, 131008, 67, 6, 31808] avg=32768.0000 max=131008 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=26 tokens_per_ep_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=128447 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=27 tokens_per_ep_rank=[32789, 0, 1222, 65244, 131008, 67, 6, 31808] avg=32768.0000 max=131008 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=28 tokens_per_ep_rank=[3, 32789, 21, 21, 130614, 33021, 12, 65663] avg=32768.0000 max=130614 max_rank=4 min=3 min_rank=0 imbalance_ratio=3.986023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=27 tokens_per_ep_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] avg=32768.0000 max=33096 max_rank=2 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97912 max_load_reduction_pct=0.747374 moved_to_replicas=130060 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=27 tokens_per_ep_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] avg=32768.0000 max=33096 max_rank=2 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97912 max_load_reduction_pct=0.747374 moved_to_replicas=130060 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[32768, 65450, 62, 65494, 32763, 32747, 32798, 62] avg=32768.0000 max=65494 max_rank=3 min=62 min_rank=2 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=27 tokens_per_ep_rank=[32789, 0, 1222, 65244, 131008, 67, 6, 31808] avg=32768.0000 max=131008 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=28 tokens_per_ep_rank=[3, 32789, 21, 21, 130614, 33021, 12, 65663] avg=32768.0000 max=130614 max_rank=4 min=3 min_rank=0 imbalance_ratio=3.986023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=27 tokens_per_ep_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] avg=32768.0000 max=33096 max_rank=2 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97912 max_load_reduction_pct=0.747374 moved_to_replicas=130060 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=28 tokens_per_ep_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] avg=32768.0000 max=33096 max_rank=4 min=32047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97518 max_load_reduction_pct=0.746612 moved_to_replicas=130085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=27 tokens_per_ep_rank=[32789, 0, 1222, 65244, 131008, 67, 6, 31808] avg=32768.0000 max=131008 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[32768, 33096, 32460, 33096, 32763, 32747, 32798, 32416] avg=32768.0000 max=33096 max_rank=1 min=32416 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=64752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=27 tokens_per_ep_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] avg=32768.0000 max=33096 max_rank=2 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97912 max_load_reduction_pct=0.747374 moved_to_replicas=130060 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=28 tokens_per_ep_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] avg=32768.0000 max=33096 max_rank=4 min=32047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97518 max_load_reduction_pct=0.746612 moved_to_replicas=130085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=28 tokens_per_ep_rank=[3, 32789, 21, 21, 130614, 33021, 12, 65663] avg=32768.0000 max=130614 max_rank=4 min=3 min_rank=0 imbalance_ratio=3.986023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=28 tokens_per_ep_rank=[3, 32789, 21, 21, 130614, 33021, 12, 65663] avg=32768.0000 max=130614 max_rank=4 min=3 min_rank=0 imbalance_ratio=3.986023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=27 tokens_per_ep_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] avg=32768.0000 max=33096 max_rank=2 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97912 max_load_reduction_pct=0.747374 moved_to_replicas=130060 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=28 tokens_per_ep_rank=[3, 32789, 21, 21, 130614, 33021, 12, 65663] avg=32768.0000 max=130614 max_rank=4 min=3 min_rank=0 imbalance_ratio=3.986023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=29 tokens_per_ep_rank=[41, 3, 65475, 21, 66280, 97536, 21, 32767] avg=32768.0000 max=97536 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.976562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=28 tokens_per_ep_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] avg=32768.0000 max=33096 max_rank=4 min=32047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97518 max_load_reduction_pct=0.746612 moved_to_replicas=130085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=28 tokens_per_ep_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] avg=32768.0000 max=33096 max_rank=4 min=32047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97518 max_load_reduction_pct=0.746612 moved_to_replicas=130085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=25 tokens_per_ep_rank=[4, 98283, 65494, 0, 20, 65513, 32788, 42] avg=32768.0000 max=98283 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=28 tokens_per_ep_rank=[3, 32789, 21, 21, 130614, 33021, 12, 65663] avg=32768.0000 max=130614 max_rank=4 min=3 min_rank=0 imbalance_ratio=3.986023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=28 tokens_per_ep_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] avg=32768.0000 max=33096 max_rank=4 min=32047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97518 max_load_reduction_pct=0.746612 moved_to_replicas=130085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=29 tokens_per_ep_rank=[41, 3, 65475, 21, 66280, 97536, 21, 32767] avg=32768.0000 max=97536 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.976562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=29 tokens_per_ep_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] avg=32768.0000 max=33096 max_rank=2 min=31715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64440 max_load_reduction_pct=0.660679 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=28 tokens_per_ep_rank=[3, 32789, 21, 21, 130614, 33021, 12, 65663] avg=32768.0000 max=130614 max_rank=4 min=3 min_rank=0 imbalance_ratio=3.986023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=25 tokens_per_ep_rank=[32445, 33096, 33096, 32746, 32437, 33096, 32788, 32440] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=130002 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=29 tokens_per_ep_rank=[41, 3, 65475, 21, 66280, 97536, 21, 32767] avg=32768.0000 max=97536 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.976562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=28 tokens_per_ep_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] avg=32768.0000 max=33096 max_rank=4 min=32047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97518 max_load_reduction_pct=0.746612 moved_to_replicas=130085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=29 tokens_per_ep_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] avg=32768.0000 max=33096 max_rank=2 min=31715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64440 max_load_reduction_pct=0.660679 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=29 tokens_per_ep_rank=[41, 3, 65475, 21, 66280, 97536, 21, 32767] avg=32768.0000 max=97536 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.976562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=28 tokens_per_ep_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] avg=32768.0000 max=33096 max_rank=4 min=32047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97518 max_load_reduction_pct=0.746612 moved_to_replicas=130085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=29 tokens_per_ep_rank=[41, 3, 65475, 21, 66280, 97536, 21, 32767] avg=32768.0000 max=97536 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.976562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=30 tokens_per_ep_rank=[32770, 65415, 32768, 32853, 32774, 21, 32754, 32789] avg=32768.0000 max=65415 max_rank=1 min=21 min_rank=5 imbalance_ratio=1.996307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=29 tokens_per_ep_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] avg=32768.0000 max=33096 max_rank=2 min=31715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64440 max_load_reduction_pct=0.660679 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=29 tokens_per_ep_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] avg=32768.0000 max=33096 max_rank=2 min=31715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64440 max_load_reduction_pct=0.660679 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=29 tokens_per_ep_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] avg=32768.0000 max=33096 max_rank=2 min=31715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64440 max_load_reduction_pct=0.660679 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=26 tokens_per_ep_rank=[64, 32768, 98238, 32742, 96401, 1917, 0, 14] avg=32768.0000 max=98238 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=29 tokens_per_ep_rank=[41, 3, 65475, 21, 66280, 97536, 21, 32767] avg=32768.0000 max=97536 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.976562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=30 tokens_per_ep_rank=[32770, 65415, 32768, 32853, 32774, 21, 32754, 32789] avg=32768.0000 max=65415 max_rank=1 min=21 min_rank=5 imbalance_ratio=1.996307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=30 tokens_per_ep_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] avg=32768.0000 max=33096 max_rank=1 min=32340 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32319 max_load_reduction_pct=0.494061 moved_to_replicas=32319 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=29 tokens_per_ep_rank=[41, 3, 65475, 21, 66280, 97536, 21, 32767] avg=32768.0000 max=97536 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.976562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=30 tokens_per_ep_rank=[32770, 65415, 32768, 32853, 32774, 21, 32754, 32789] avg=32768.0000 max=65415 max_rank=1 min=21 min_rank=5 imbalance_ratio=1.996307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=26 tokens_per_ep_rank=[32810, 32768, 33096, 32742, 33096, 32476, 32746, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=128447 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=29 tokens_per_ep_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] avg=32768.0000 max=33096 max_rank=2 min=31715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64440 max_load_reduction_pct=0.660679 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=30 tokens_per_ep_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] avg=32768.0000 max=33096 max_rank=1 min=32340 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32319 max_load_reduction_pct=0.494061 moved_to_replicas=32319 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=30 tokens_per_ep_rank=[32770, 65415, 32768, 32853, 32774, 21, 32754, 32789] avg=32768.0000 max=65415 max_rank=1 min=21 min_rank=5 imbalance_ratio=1.996307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=29 tokens_per_ep_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] avg=32768.0000 max=33096 max_rank=2 min=31715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64440 max_load_reduction_pct=0.660679 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=30 tokens_per_ep_rank=[32770, 65415, 32768, 32853, 32774, 21, 32754, 32789] avg=32768.0000 max=65415 max_rank=1 min=21 min_rank=5 imbalance_ratio=1.996307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=31 tokens_per_ep_rank=[80, 21, 97710, 65516, 65510, 1, 20, 33286] avg=32768.0000 max=97710 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.981873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=30 tokens_per_ep_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] avg=32768.0000 max=33096 max_rank=1 min=32340 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32319 max_load_reduction_pct=0.494061 moved_to_replicas=32319 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=30 tokens_per_ep_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] avg=32768.0000 max=33096 max_rank=1 min=32340 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32319 max_load_reduction_pct=0.494061 moved_to_replicas=32319 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=30 tokens_per_ep_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] avg=32768.0000 max=33096 max_rank=1 min=32340 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32319 max_load_reduction_pct=0.494061 moved_to_replicas=32319 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=27 tokens_per_ep_rank=[32789, 0, 1222, 65244, 131008, 67, 6, 31808] avg=32768.0000 max=131008 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=31 tokens_per_ep_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] avg=32768.0000 max=33096 max_rank=2 min=32078 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64614 max_load_reduction_pct=0.661283 moved_to_replicas=129638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=30 tokens_per_ep_rank=[32770, 65415, 32768, 32853, 32774, 21, 32754, 32789] avg=32768.0000 max=65415 max_rank=1 min=21 min_rank=5 imbalance_ratio=1.996307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=31 tokens_per_ep_rank=[80, 21, 97710, 65516, 65510, 1, 20, 33286] avg=32768.0000 max=97710 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.981873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=30 tokens_per_ep_rank=[32770, 65415, 32768, 32853, 32774, 21, 32754, 32789] avg=32768.0000 max=65415 max_rank=1 min=21 min_rank=5 imbalance_ratio=1.996307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=31 tokens_per_ep_rank=[80, 21, 97710, 65516, 65510, 1, 20, 33286] avg=32768.0000 max=97710 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.981873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=27 tokens_per_ep_rank=[32789, 32746, 33096, 33096, 33096, 32487, 32752, 32082] avg=32768.0000 max=33096 max_rank=2 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97912 max_load_reduction_pct=0.747374 moved_to_replicas=130060 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=30 tokens_per_ep_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] avg=32768.0000 max=33096 max_rank=1 min=32340 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32319 max_load_reduction_pct=0.494061 moved_to_replicas=32319 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=31 tokens_per_ep_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] avg=32768.0000 max=33096 max_rank=2 min=32078 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64614 max_load_reduction_pct=0.661283 moved_to_replicas=129638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=31 tokens_per_ep_rank=[80, 21, 97710, 65516, 65510, 1, 20, 33286] avg=32768.0000 max=97710 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.981873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=31 tokens_per_ep_rank=[80, 21, 97710, 65516, 65510, 1, 20, 33286] avg=32768.0000 max=97710 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.981873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=30 tokens_per_ep_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] avg=32768.0000 max=33096 max_rank=1 min=32340 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32319 max_load_reduction_pct=0.494061 moved_to_replicas=32319 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=32 tokens_per_ep_rank=[65553, 32768, 32745, 65330, 32937, 44, 32767, 0] avg=32768.0000 max=65553 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.000519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=31 tokens_per_ep_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] avg=32768.0000 max=33096 max_rank=2 min=32078 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64614 max_load_reduction_pct=0.661283 moved_to_replicas=129638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=31 tokens_per_ep_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] avg=32768.0000 max=33096 max_rank=2 min=32078 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64614 max_load_reduction_pct=0.661283 moved_to_replicas=129638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=31 tokens_per_ep_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] avg=32768.0000 max=33096 max_rank=2 min=32078 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64614 max_load_reduction_pct=0.661283 moved_to_replicas=129638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=28 tokens_per_ep_rank=[3, 32789, 21, 21, 130614, 33021, 12, 65663] avg=32768.0000 max=130614 max_rank=4 min=3 min_rank=0 imbalance_ratio=3.986023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=32 tokens_per_ep_rank=[65553, 32768, 32745, 65330, 32937, 44, 32767, 0] avg=32768.0000 max=65553 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.000519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=32 tokens_per_ep_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] avg=32768.0000 max=33096 max_rank=0 min=32278 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32457 max_load_reduction_pct=0.495126 moved_to_replicas=64691 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=31 tokens_per_ep_rank=[80, 21, 97710, 65516, 65510, 1, 20, 33286] avg=32768.0000 max=97710 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.981873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=31 tokens_per_ep_rank=[80, 21, 97710, 65516, 65510, 1, 20, 33286] avg=32768.0000 max=97710 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.981873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=32 tokens_per_ep_rank=[65553, 32768, 32745, 65330, 32937, 44, 32767, 0] avg=32768.0000 max=65553 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.000519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=28 tokens_per_ep_rank=[32749, 32789, 32047, 32588, 33096, 33021, 32758, 33096] avg=32768.0000 max=33096 max_rank=4 min=32047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97518 max_load_reduction_pct=0.746612 moved_to_replicas=130085 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=32 tokens_per_ep_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] avg=32768.0000 max=33096 max_rank=0 min=32278 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32457 max_load_reduction_pct=0.495126 moved_to_replicas=64691 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=31 tokens_per_ep_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] avg=32768.0000 max=33096 max_rank=2 min=32078 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64614 max_load_reduction_pct=0.661283 moved_to_replicas=129638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=31 tokens_per_ep_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] avg=32768.0000 max=33096 max_rank=2 min=32078 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64614 max_load_reduction_pct=0.661283 moved_to_replicas=129638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=32 tokens_per_ep_rank=[65553, 32768, 32745, 65330, 32937, 44, 32767, 0] avg=32768.0000 max=65553 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.000519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=32 tokens_per_ep_rank=[65553, 32768, 32745, 65330, 32937, 44, 32767, 0] avg=32768.0000 max=65553 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.000519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=33 tokens_per_ep_rank=[65588, 80, 32741, 32793, 31075, 65491, 34365, 11] avg=32768.0000 max=65588 max_rank=0 min=11 min_rank=7 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=32 tokens_per_ep_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] avg=32768.0000 max=33096 max_rank=0 min=32278 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32457 max_load_reduction_pct=0.495126 moved_to_replicas=64691 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=32 tokens_per_ep_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] avg=32768.0000 max=33096 max_rank=0 min=32278 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32457 max_load_reduction_pct=0.495126 moved_to_replicas=64691 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=32 tokens_per_ep_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] avg=32768.0000 max=33096 max_rank=0 min=32278 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32457 max_load_reduction_pct=0.495126 moved_to_replicas=64691 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=29 tokens_per_ep_rank=[41, 3, 65475, 21, 66280, 97536, 21, 32767] avg=32768.0000 max=97536 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.976562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=33 tokens_per_ep_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] avg=32768.0000 max=33096 max_rank=0 min=32344 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32492 max_load_reduction_pct=0.495395 moved_to_replicas=66156 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=33 tokens_per_ep_rank=[65588, 80, 32741, 32793, 31075, 65491, 34365, 11] avg=32768.0000 max=65588 max_rank=0 min=11 min_rank=7 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=32 tokens_per_ep_rank=[65553, 32768, 32745, 65330, 32937, 44, 32767, 0] avg=32768.0000 max=65553 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.000519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=32 tokens_per_ep_rank=[65553, 32768, 32745, 65330, 32937, 44, 32767, 0] avg=32768.0000 max=65553 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.000519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=33 tokens_per_ep_rank=[65588, 80, 32741, 32793, 31075, 65491, 34365, 11] avg=32768.0000 max=65588 max_rank=0 min=11 min_rank=7 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=29 tokens_per_ep_rank=[32858, 32749, 33096, 31715, 33096, 33096, 32767, 32767] avg=32768.0000 max=33096 max_rank=2 min=31715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64440 max_load_reduction_pct=0.660679 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=33 tokens_per_ep_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] avg=32768.0000 max=33096 max_rank=0 min=32344 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32492 max_load_reduction_pct=0.495395 moved_to_replicas=66156 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=32 tokens_per_ep_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] avg=32768.0000 max=33096 max_rank=0 min=32278 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32457 max_load_reduction_pct=0.495126 moved_to_replicas=64691 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=32 tokens_per_ep_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] avg=32768.0000 max=33096 max_rank=0 min=32278 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32457 max_load_reduction_pct=0.495126 moved_to_replicas=64691 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=33 tokens_per_ep_rank=[65588, 80, 32741, 32793, 31075, 65491, 34365, 11] avg=32768.0000 max=65588 max_rank=0 min=11 min_rank=7 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=33 tokens_per_ep_rank=[65588, 80, 32741, 32793, 31075, 65491, 34365, 11] avg=32768.0000 max=65588 max_rank=0 min=11 min_rank=7 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=34 tokens_per_ep_rank=[98131, 65426, 65504, 22, 64, 0, 57, 32940] avg=32768.0000 max=98131 max_rank=0 min=0 min_rank=5 imbalance_ratio=2.994720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=33 tokens_per_ep_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] avg=32768.0000 max=33096 max_rank=0 min=32344 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32492 max_load_reduction_pct=0.495395 moved_to_replicas=66156 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=33 tokens_per_ep_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] avg=32768.0000 max=33096 max_rank=0 min=32344 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32492 max_load_reduction_pct=0.495395 moved_to_replicas=66156 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=33 tokens_per_ep_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] avg=32768.0000 max=33096 max_rank=0 min=32344 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32492 max_load_reduction_pct=0.495395 moved_to_replicas=66156 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65035 max_load_reduction_pct=0.662737 moved_to_replicas=129773 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=30 tokens_per_ep_rank=[32770, 65415, 32768, 32853, 32774, 21, 32754, 32789] avg=32768.0000 max=65415 max_rank=1 min=21 min_rank=5 imbalance_ratio=1.996307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=34 tokens_per_ep_rank=[98131, 65426, 65504, 22, 64, 0, 57, 32940] avg=32768.0000 max=98131 max_rank=0 min=0 min_rank=5 imbalance_ratio=2.994720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=33 tokens_per_ep_rank=[65588, 80, 32741, 32793, 31075, 65491, 34365, 11] avg=32768.0000 max=65588 max_rank=0 min=11 min_rank=7 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=33 tokens_per_ep_rank=[65588, 80, 32741, 32793, 31075, 65491, 34365, 11] avg=32768.0000 max=65588 max_rank=0 min=11 min_rank=7 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=34 tokens_per_ep_rank=[98131, 65426, 65504, 22, 64, 0, 57, 32940] avg=32768.0000 max=98131 max_rank=0 min=0 min_rank=5 imbalance_ratio=2.994720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=30 tokens_per_ep_rank=[32770, 33096, 32768, 32853, 32774, 32340, 32754, 32789] avg=32768.0000 max=33096 max_rank=1 min=32340 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32319 max_load_reduction_pct=0.494061 moved_to_replicas=32319 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65035 max_load_reduction_pct=0.662737 moved_to_replicas=129773 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=33 tokens_per_ep_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] avg=32768.0000 max=33096 max_rank=0 min=32344 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32492 max_load_reduction_pct=0.495395 moved_to_replicas=66156 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=33 tokens_per_ep_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] avg=32768.0000 max=33096 max_rank=0 min=32344 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32492 max_load_reduction_pct=0.495395 moved_to_replicas=66156 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=34 tokens_per_ep_rank=[98131, 65426, 65504, 22, 64, 0, 57, 32940] avg=32768.0000 max=98131 max_rank=0 min=0 min_rank=5 imbalance_ratio=2.994720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=34 tokens_per_ep_rank=[98131, 65426, 65504, 22, 64, 0, 57, 32940] avg=32768.0000 max=98131 max_rank=0 min=0 min_rank=5 imbalance_ratio=2.994720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=35 tokens_per_ep_rank=[32773, 21, 32791, 130757, 32747, 262, 32793, 0] avg=32768.0000 max=130757 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.990387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65035 max_load_reduction_pct=0.662737 moved_to_replicas=129773 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65035 max_load_reduction_pct=0.662737 moved_to_replicas=129773 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65035 max_load_reduction_pct=0.662737 moved_to_replicas=129773 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=35 tokens_per_ep_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] avg=32768.0000 max=33096 max_rank=3 min=32431 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97661 max_load_reduction_pct=0.746889 moved_to_replicas=97661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=31 tokens_per_ep_rank=[80, 21, 97710, 65516, 65510, 1, 20, 33286] avg=32768.0000 max=97710 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.981873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=35 tokens_per_ep_rank=[32773, 21, 32791, 130757, 32747, 262, 32793, 0] avg=32768.0000 max=130757 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.990387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=34 tokens_per_ep_rank=[98131, 65426, 65504, 22, 64, 0, 57, 32940] avg=32768.0000 max=98131 max_rank=0 min=0 min_rank=5 imbalance_ratio=2.994720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=34 tokens_per_ep_rank=[98131, 65426, 65504, 22, 64, 0, 57, 32940] avg=32768.0000 max=98131 max_rank=0 min=0 min_rank=5 imbalance_ratio=2.994720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=35 tokens_per_ep_rank=[32773, 21, 32791, 130757, 32747, 262, 32793, 0] avg=32768.0000 max=130757 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.990387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=31 tokens_per_ep_rank=[32494, 32441, 33096, 33096, 33096, 32747, 32078, 33096] avg=32768.0000 max=33096 max_rank=2 min=32078 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64614 max_load_reduction_pct=0.661283 moved_to_replicas=129638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=35 tokens_per_ep_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] avg=32768.0000 max=33096 max_rank=3 min=32431 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97661 max_load_reduction_pct=0.746889 moved_to_replicas=97661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=35 tokens_per_ep_rank=[32773, 21, 32791, 130757, 32747, 262, 32793, 0] avg=32768.0000 max=130757 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.990387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=36 tokens_per_ep_rank=[18829, 101184, 32747, 32729, 21, 65561, 21, 11052] avg=32768.0000 max=101184 max_rank=1 min=21 min_rank=4 imbalance_ratio=3.087891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65035 max_load_reduction_pct=0.662737 moved_to_replicas=129773 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=35 tokens_per_ep_rank=[32773, 21, 32791, 130757, 32747, 262, 32793, 0] avg=32768.0000 max=130757 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.990387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65035 max_load_reduction_pct=0.662737 moved_to_replicas=129773 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=35 tokens_per_ep_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] avg=32768.0000 max=33096 max_rank=3 min=32431 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97661 max_load_reduction_pct=0.746889 moved_to_replicas=97661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=35 tokens_per_ep_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] avg=32768.0000 max=33096 max_rank=3 min=32431 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97661 max_load_reduction_pct=0.746889 moved_to_replicas=97661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=36 tokens_per_ep_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=31846 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68088 max_load_reduction_pct=0.672913 moved_to_replicas=100553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=35 tokens_per_ep_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] avg=32768.0000 max=33096 max_rank=3 min=32431 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97661 max_load_reduction_pct=0.746889 moved_to_replicas=97661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=32 tokens_per_ep_rank=[65553, 32768, 32745, 65330, 32937, 44, 32767, 0] avg=32768.0000 max=65553 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.000519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=36 tokens_per_ep_rank=[18829, 101184, 32747, 32729, 21, 65561, 21, 11052] avg=32768.0000 max=101184 max_rank=1 min=21 min_rank=4 imbalance_ratio=3.087891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=35 tokens_per_ep_rank=[32773, 21, 32791, 130757, 32747, 262, 32793, 0] avg=32768.0000 max=130757 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.990387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=36 tokens_per_ep_rank=[18829, 101184, 32747, 32729, 21, 65561, 21, 11052] avg=32768.0000 max=101184 max_rank=1 min=21 min_rank=4 imbalance_ratio=3.087891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=35 tokens_per_ep_rank=[32773, 21, 32791, 130757, 32747, 262, 32793, 0] avg=32768.0000 max=130757 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.990387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=36 tokens_per_ep_rank=[18829, 101184, 32747, 32729, 21, 65561, 21, 11052] avg=32768.0000 max=101184 max_rank=1 min=21 min_rank=4 imbalance_ratio=3.087891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=32 tokens_per_ep_rank=[33096, 32768, 32745, 33096, 32937, 32278, 32767, 32457] avg=32768.0000 max=33096 max_rank=0 min=32278 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32457 max_load_reduction_pct=0.495126 moved_to_replicas=64691 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=36 tokens_per_ep_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=31846 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68088 max_load_reduction_pct=0.672913 moved_to_replicas=100553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=35 tokens_per_ep_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] avg=32768.0000 max=33096 max_rank=3 min=32431 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97661 max_load_reduction_pct=0.746889 moved_to_replicas=97661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 706, 0, 64595, 65574] avg=32768.0000 max=65574 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.001160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=36 tokens_per_ep_rank=[18829, 101184, 32747, 32729, 21, 65561, 21, 11052] avg=32768.0000 max=101184 max_rank=1 min=21 min_rank=4 imbalance_ratio=3.087891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=36 tokens_per_ep_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=31846 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68088 max_load_reduction_pct=0.672913 moved_to_replicas=100553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=35 tokens_per_ep_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] avg=32768.0000 max=33096 max_rank=3 min=32431 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97661 max_load_reduction_pct=0.746889 moved_to_replicas=97661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=36 tokens_per_ep_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=31846 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68088 max_load_reduction_pct=0.672913 moved_to_replicas=100553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32205 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32478 max_load_reduction_pct=0.495288 moved_to_replicas=63977 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=36 tokens_per_ep_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=31846 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68088 max_load_reduction_pct=0.672913 moved_to_replicas=100553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=33 tokens_per_ep_rank=[65588, 80, 32741, 32793, 31075, 65491, 34365, 11] avg=32768.0000 max=65588 max_rank=0 min=11 min_rank=7 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 706, 0, 64595, 65574] avg=32768.0000 max=65574 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.001160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=36 tokens_per_ep_rank=[18829, 101184, 32747, 32729, 21, 65561, 21, 11052] avg=32768.0000 max=101184 max_rank=1 min=21 min_rank=4 imbalance_ratio=3.087891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 706, 0, 64595, 65574] avg=32768.0000 max=65574 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.001160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=36 tokens_per_ep_rank=[18829, 101184, 32747, 32729, 21, 65561, 21, 11052] avg=32768.0000 max=101184 max_rank=1 min=21 min_rank=4 imbalance_ratio=3.087891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 706, 0, 64595, 65574] avg=32768.0000 max=65574 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.001160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=33 tokens_per_ep_rank=[33096, 32475, 32741, 32793, 32344, 33096, 33096, 32503] avg=32768.0000 max=33096 max_rank=0 min=32344 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32492 max_load_reduction_pct=0.495395 moved_to_replicas=66156 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 130988, 306, 32673, 0, 25, 32768] avg=32768.0000 max=130988 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32205 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32478 max_load_reduction_pct=0.495288 moved_to_replicas=63977 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 706, 0, 64595, 65574] avg=32768.0000 max=65574 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.001160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=36 tokens_per_ep_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=31846 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68088 max_load_reduction_pct=0.672913 moved_to_replicas=100553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32205 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32478 max_load_reduction_pct=0.495288 moved_to_replicas=63977 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=36 tokens_per_ep_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=31846 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68088 max_load_reduction_pct=0.672913 moved_to_replicas=100553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32205 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32478 max_load_reduction_pct=0.495288 moved_to_replicas=63977 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] avg=32768.0000 max=33096 max_rank=2 min=32616 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97892 max_load_reduction_pct=0.747336 moved_to_replicas=97892 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32205 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32478 max_load_reduction_pct=0.495288 moved_to_replicas=63977 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=34 tokens_per_ep_rank=[98131, 65426, 65504, 22, 64, 0, 57, 32940] avg=32768.0000 max=98131 max_rank=0 min=0 min_rank=5 imbalance_ratio=2.994720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 130988, 306, 32673, 0, 25, 32768] avg=32768.0000 max=130988 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 706, 0, 64595, 65574] avg=32768.0000 max=65574 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.001160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 130988, 306, 32673, 0, 25, 32768] avg=32768.0000 max=130988 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 706, 0, 64595, 65574] avg=32768.0000 max=65574 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.001160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 130988, 306, 32673, 0, 25, 32768] avg=32768.0000 max=130988 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=39 tokens_per_ep_rank=[65515, 65495, 65, 32811, 22, 32780, 32962, 32494] avg=32768.0000 max=65515 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32312, 32394, 32745, 32465, 32940] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65035 max_load_reduction_pct=0.662737 moved_to_replicas=129773 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] avg=32768.0000 max=33096 max_rank=2 min=32616 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97892 max_load_reduction_pct=0.747336 moved_to_replicas=97892 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 130988, 306, 32673, 0, 25, 32768] avg=32768.0000 max=130988 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32205 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32478 max_load_reduction_pct=0.495288 moved_to_replicas=63977 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] avg=32768.0000 max=33096 max_rank=2 min=32616 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97892 max_load_reduction_pct=0.747336 moved_to_replicas=97892 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32205 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32478 max_load_reduction_pct=0.495288 moved_to_replicas=63977 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] avg=32768.0000 max=33096 max_rank=2 min=32616 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97892 max_load_reduction_pct=0.747336 moved_to_replicas=97892 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=39 tokens_per_ep_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] avg=32768.0000 max=33096 max_rank=0 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64818 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] avg=32768.0000 max=33096 max_rank=2 min=32616 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97892 max_load_reduction_pct=0.747336 moved_to_replicas=97892 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=35 tokens_per_ep_rank=[32773, 21, 32791, 130757, 32747, 262, 32793, 0] avg=32768.0000 max=130757 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.990387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=39 tokens_per_ep_rank=[65515, 65495, 65, 32811, 22, 32780, 32962, 32494] avg=32768.0000 max=65515 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 130988, 306, 32673, 0, 25, 32768] avg=32768.0000 max=130988 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=39 tokens_per_ep_rank=[65515, 65495, 65, 32811, 22, 32780, 32962, 32494] avg=32768.0000 max=65515 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=39 tokens_per_ep_rank=[65515, 65495, 65, 32811, 22, 32780, 32962, 32494] avg=32768.0000 max=65515 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 130988, 306, 32673, 0, 25, 32768] avg=32768.0000 max=130988 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=40 tokens_per_ep_rank=[21, 60081, 70915, 65515, 32, 32770, 32789, 21] avg=32768.0000 max=70915 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.164154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=39 tokens_per_ep_rank=[65515, 65495, 65, 32811, 22, 32780, 32962, 32494] avg=32768.0000 max=65515 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=35 tokens_per_ep_rank=[32773, 32767, 32791, 33096, 32747, 32431, 32793, 32746] avg=32768.0000 max=33096 max_rank=3 min=32431 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97661 max_load_reduction_pct=0.746889 moved_to_replicas=97661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=39 tokens_per_ep_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] avg=32768.0000 max=33096 max_rank=0 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64818 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=39 tokens_per_ep_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] avg=32768.0000 max=33096 max_rank=0 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64818 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] avg=32768.0000 max=33096 max_rank=2 min=32616 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97892 max_load_reduction_pct=0.747336 moved_to_replicas=97892 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=39 tokens_per_ep_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] avg=32768.0000 max=33096 max_rank=0 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64818 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] avg=32768.0000 max=33096 max_rank=2 min=32616 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97892 max_load_reduction_pct=0.747336 moved_to_replicas=97892 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=40 tokens_per_ep_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] avg=32768.0000 max=33097 max_rank=1 min=32077 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37818 max_load_reduction_pct=0.533286 moved_to_replicas=97223 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=39 tokens_per_ep_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] avg=32768.0000 max=33096 max_rank=0 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64818 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=36 tokens_per_ep_rank=[18829, 101184, 32747, 32729, 21, 65561, 21, 11052] avg=32768.0000 max=101184 max_rank=1 min=21 min_rank=4 imbalance_ratio=3.087891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=40 tokens_per_ep_rank=[21, 60081, 70915, 65515, 32, 32770, 32789, 21] avg=32768.0000 max=70915 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.164154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=40 tokens_per_ep_rank=[21, 60081, 70915, 65515, 32, 32770, 32789, 21] avg=32768.0000 max=70915 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.164154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=39 tokens_per_ep_rank=[65515, 65495, 65, 32811, 22, 32780, 32962, 32494] avg=32768.0000 max=65515 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=40 tokens_per_ep_rank=[21, 60081, 70915, 65515, 32, 32770, 32789, 21] avg=32768.0000 max=70915 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.164154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=39 tokens_per_ep_rank=[65515, 65495, 65, 32811, 22, 32780, 32962, 32494] avg=32768.0000 max=65515 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=41 tokens_per_ep_rank=[113, 32768, 32768, 65494, 65247, 177, 83, 65494] avg=32768.0000 max=65494 max_rank=3 min=83 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=40 tokens_per_ep_rank=[21, 60081, 70915, 65515, 32, 32770, 32789, 21] avg=32768.0000 max=70915 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.164154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=36 tokens_per_ep_rank=[31846, 33096, 32747, 32729, 32767, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=31846 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=68088 max_load_reduction_pct=0.672913 moved_to_replicas=100553 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=40 tokens_per_ep_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] avg=32768.0000 max=33097 max_rank=1 min=32077 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37818 max_load_reduction_pct=0.533286 moved_to_replicas=97223 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=40 tokens_per_ep_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] avg=32768.0000 max=33097 max_rank=1 min=32077 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37818 max_load_reduction_pct=0.533286 moved_to_replicas=97223 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=39 tokens_per_ep_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] avg=32768.0000 max=33096 max_rank=0 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64818 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=40 tokens_per_ep_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] avg=32768.0000 max=33097 max_rank=1 min=32077 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37818 max_load_reduction_pct=0.533286 moved_to_replicas=97223 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=39 tokens_per_ep_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] avg=32768.0000 max=33096 max_rank=0 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64818 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=41 tokens_per_ep_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] avg=32768.0000 max=33096 max_rank=3 min=32328 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=96947 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=40 tokens_per_ep_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] avg=32768.0000 max=33097 max_rank=1 min=32077 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37818 max_load_reduction_pct=0.533286 moved_to_replicas=97223 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 706, 0, 64595, 65574] avg=32768.0000 max=65574 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.001160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=41 tokens_per_ep_rank=[113, 32768, 32768, 65494, 65247, 177, 83, 65494] avg=32768.0000 max=65494 max_rank=3 min=83 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=41 tokens_per_ep_rank=[113, 32768, 32768, 65494, 65247, 177, 83, 65494] avg=32768.0000 max=65494 max_rank=3 min=83 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=40 tokens_per_ep_rank=[21, 60081, 70915, 65515, 32, 32770, 32789, 21] avg=32768.0000 max=70915 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.164154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=41 tokens_per_ep_rank=[113, 32768, 32768, 65494, 65247, 177, 83, 65494] avg=32768.0000 max=65494 max_rank=3 min=83 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=40 tokens_per_ep_rank=[21, 60081, 70915, 65515, 32, 32770, 32789, 21] avg=32768.0000 max=70915 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.164154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=42 tokens_per_ep_rank=[32770, 65515, 29101, 65514, 42, 32773, 3683, 32746] avg=32768.0000 max=65515 max_rank=1 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=41 tokens_per_ep_rank=[113, 32768, 32768, 65494, 65247, 177, 83, 65494] avg=32768.0000 max=65494 max_rank=3 min=83 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=37 tokens_per_ep_rank=[32902, 32810, 32794, 32763, 32205, 32478, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32205 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32478 max_load_reduction_pct=0.495288 moved_to_replicas=63977 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=41 tokens_per_ep_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] avg=32768.0000 max=33096 max_rank=3 min=32328 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=96947 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=41 tokens_per_ep_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] avg=32768.0000 max=33096 max_rank=3 min=32328 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=96947 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=41 tokens_per_ep_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] avg=32768.0000 max=33096 max_rank=3 min=32328 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=96947 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=40 tokens_per_ep_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] avg=32768.0000 max=33097 max_rank=1 min=32077 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37818 max_load_reduction_pct=0.533286 moved_to_replicas=97223 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=40 tokens_per_ep_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] avg=32768.0000 max=33097 max_rank=1 min=32077 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37818 max_load_reduction_pct=0.533286 moved_to_replicas=97223 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=42 tokens_per_ep_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] avg=32768.0000 max=33096 max_rank=1 min=32106 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64837 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=41 tokens_per_ep_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] avg=32768.0000 max=33096 max_rank=3 min=32328 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=96947 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 130988, 306, 32673, 0, 25, 32768] avg=32768.0000 max=130988 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=42 tokens_per_ep_rank=[32770, 65515, 29101, 65514, 42, 32773, 3683, 32746] avg=32768.0000 max=65515 max_rank=1 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=42 tokens_per_ep_rank=[32770, 65515, 29101, 65514, 42, 32773, 3683, 32746] avg=32768.0000 max=65515 max_rank=1 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=42 tokens_per_ep_rank=[32770, 65515, 29101, 65514, 42, 32773, 3683, 32746] avg=32768.0000 max=65515 max_rank=1 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=41 tokens_per_ep_rank=[113, 32768, 32768, 65494, 65247, 177, 83, 65494] avg=32768.0000 max=65494 max_rank=3 min=83 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=41 tokens_per_ep_rank=[113, 32768, 32768, 65494, 65247, 177, 83, 65494] avg=32768.0000 max=65494 max_rank=3 min=83 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=43 tokens_per_ep_rank=[64929, 79081, 21, 42, 43, 95995, 0, 22033] avg=32768.0000 max=95995 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.929535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=42 tokens_per_ep_rank=[32770, 65515, 29101, 65514, 42, 32773, 3683, 32746] avg=32768.0000 max=65515 max_rank=1 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=38 tokens_per_ep_rank=[32768, 32616, 33096, 32706, 32673, 32746, 32771, 32768] avg=32768.0000 max=33096 max_rank=2 min=32616 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97892 max_load_reduction_pct=0.747336 moved_to_replicas=97892 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=42 tokens_per_ep_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] avg=32768.0000 max=33096 max_rank=1 min=32106 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64837 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=42 tokens_per_ep_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] avg=32768.0000 max=33096 max_rank=1 min=32106 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64837 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=42 tokens_per_ep_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] avg=32768.0000 max=33096 max_rank=1 min=32106 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64837 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=41 tokens_per_ep_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] avg=32768.0000 max=33096 max_rank=3 min=32328 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=96947 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=41 tokens_per_ep_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] avg=32768.0000 max=33096 max_rank=3 min=32328 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=96947 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=43 tokens_per_ep_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] avg=32768.0000 max=33096 max_rank=0 min=31130 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62899 max_load_reduction_pct=0.655232 moved_to_replicas=140717 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=42 tokens_per_ep_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] avg=32768.0000 max=33096 max_rank=1 min=32106 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64837 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=43 tokens_per_ep_rank=[64929, 79081, 21, 42, 43, 95995, 0, 22033] avg=32768.0000 max=95995 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.929535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=39 tokens_per_ep_rank=[65515, 65495, 65, 32811, 22, 32780, 32962, 32494] avg=32768.0000 max=65515 max_rank=0 min=22 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=43 tokens_per_ep_rank=[64929, 79081, 21, 42, 43, 95995, 0, 22033] avg=32768.0000 max=95995 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.929535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=43 tokens_per_ep_rank=[64929, 79081, 21, 42, 43, 95995, 0, 22033] avg=32768.0000 max=95995 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.929535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=42 tokens_per_ep_rank=[32770, 65515, 29101, 65514, 42, 32773, 3683, 32746] avg=32768.0000 max=65515 max_rank=1 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=42 tokens_per_ep_rank=[32770, 65515, 29101, 65514, 42, 32773, 3683, 32746] avg=32768.0000 max=65515 max_rank=1 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=44 tokens_per_ep_rank=[51, 42, 32747, 32780, 32724, 65502, 98229, 69] avg=32768.0000 max=98229 max_rank=6 min=42 min_rank=1 imbalance_ratio=2.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=43 tokens_per_ep_rank=[64929, 79081, 21, 42, 43, 95995, 0, 22033] avg=32768.0000 max=95995 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.929535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=43 tokens_per_ep_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] avg=32768.0000 max=33096 max_rank=0 min=31130 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62899 max_load_reduction_pct=0.655232 moved_to_replicas=140717 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=39 tokens_per_ep_rank=[33096, 33096, 32464, 32811, 32441, 32780, 32962, 32494] avg=32768.0000 max=33096 max_rank=0 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64818 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=43 tokens_per_ep_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] avg=32768.0000 max=33096 max_rank=0 min=31130 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62899 max_load_reduction_pct=0.655232 moved_to_replicas=140717 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=43 tokens_per_ep_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] avg=32768.0000 max=33096 max_rank=0 min=31130 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62899 max_load_reduction_pct=0.655232 moved_to_replicas=140717 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=42 tokens_per_ep_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] avg=32768.0000 max=33096 max_rank=1 min=32106 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64837 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=42 tokens_per_ep_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] avg=32768.0000 max=33096 max_rank=1 min=32106 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64837 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=44 tokens_per_ep_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] avg=32768.0000 max=33096 max_rank=5 min=32438 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65133 max_load_reduction_pct=0.663073 moved_to_replicas=97539 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=43 tokens_per_ep_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] avg=32768.0000 max=33096 max_rank=0 min=31130 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62899 max_load_reduction_pct=0.655232 moved_to_replicas=140717 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=44 tokens_per_ep_rank=[51, 42, 32747, 32780, 32724, 65502, 98229, 69] avg=32768.0000 max=98229 max_rank=6 min=42 min_rank=1 imbalance_ratio=2.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=44 tokens_per_ep_rank=[51, 42, 32747, 32780, 32724, 65502, 98229, 69] avg=32768.0000 max=98229 max_rank=6 min=42 min_rank=1 imbalance_ratio=2.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=44 tokens_per_ep_rank=[51, 42, 32747, 32780, 32724, 65502, 98229, 69] avg=32768.0000 max=98229 max_rank=6 min=42 min_rank=1 imbalance_ratio=2.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=43 tokens_per_ep_rank=[64929, 79081, 21, 42, 43, 95995, 0, 22033] avg=32768.0000 max=95995 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.929535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=40 tokens_per_ep_rank=[21, 60081, 70915, 65515, 32, 32770, 32789, 21] avg=32768.0000 max=70915 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.164154 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=43 tokens_per_ep_rank=[64929, 79081, 21, 42, 43, 95995, 0, 22033] avg=32768.0000 max=95995 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.929535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=45 tokens_per_ep_rank=[42, 32768, 4, 65487, 98294, 65486, 21, 42] avg=32768.0000 max=98294 max_rank=4 min=4 min_rank=2 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=44 tokens_per_ep_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] avg=32768.0000 max=33096 max_rank=5 min=32438 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65133 max_load_reduction_pct=0.663073 moved_to_replicas=97539 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=44 tokens_per_ep_rank=[51, 42, 32747, 32780, 32724, 65502, 98229, 69] avg=32768.0000 max=98229 max_rank=6 min=42 min_rank=1 imbalance_ratio=2.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=44 tokens_per_ep_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] avg=32768.0000 max=33096 max_rank=5 min=32438 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65133 max_load_reduction_pct=0.663073 moved_to_replicas=97539 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=44 tokens_per_ep_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] avg=32768.0000 max=33096 max_rank=5 min=32438 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65133 max_load_reduction_pct=0.663073 moved_to_replicas=97539 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=43 tokens_per_ep_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] avg=32768.0000 max=33096 max_rank=0 min=31130 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62899 max_load_reduction_pct=0.655232 moved_to_replicas=140717 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=40 tokens_per_ep_rank=[32767, 33097, 33097, 33096, 32451, 32770, 32789, 32077] avg=32768.0000 max=33097 max_rank=1 min=32077 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37818 max_load_reduction_pct=0.533286 moved_to_replicas=97223 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=43 tokens_per_ep_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] avg=32768.0000 max=33096 max_rank=0 min=31130 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62899 max_load_reduction_pct=0.655232 moved_to_replicas=140717 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=45 tokens_per_ep_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] avg=32768.0000 max=33096 max_rank=3 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=129979 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=44 tokens_per_ep_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] avg=32768.0000 max=33096 max_rank=5 min=32438 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65133 max_load_reduction_pct=0.663073 moved_to_replicas=97539 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=45 tokens_per_ep_rank=[42, 32768, 4, 65487, 98294, 65486, 21, 42] avg=32768.0000 max=98294 max_rank=4 min=4 min_rank=2 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=45 tokens_per_ep_rank=[42, 32768, 4, 65487, 98294, 65486, 21, 42] avg=32768.0000 max=98294 max_rank=4 min=4 min_rank=2 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=45 tokens_per_ep_rank=[42, 32768, 4, 65487, 98294, 65486, 21, 42] avg=32768.0000 max=98294 max_rank=4 min=4 min_rank=2 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=44 tokens_per_ep_rank=[51, 42, 32747, 32780, 32724, 65502, 98229, 69] avg=32768.0000 max=98229 max_rank=6 min=42 min_rank=1 imbalance_ratio=2.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=41 tokens_per_ep_rank=[113, 32768, 32768, 65494, 65247, 177, 83, 65494] avg=32768.0000 max=65494 max_rank=3 min=83 min_rank=6 imbalance_ratio=1.998718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=44 tokens_per_ep_rank=[51, 42, 32747, 32780, 32724, 65502, 98229, 69] avg=32768.0000 max=98229 max_rank=6 min=42 min_rank=1 imbalance_ratio=2.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=46 tokens_per_ep_rank=[0, 31, 34, 32747, 32763, 32789, 130998, 32782] avg=32768.0000 max=130998 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=45 tokens_per_ep_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] avg=32768.0000 max=33096 max_rank=3 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=129979 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=45 tokens_per_ep_rank=[42, 32768, 4, 65487, 98294, 65486, 21, 42] avg=32768.0000 max=98294 max_rank=4 min=4 min_rank=2 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=45 tokens_per_ep_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] avg=32768.0000 max=33096 max_rank=3 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=129979 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=45 tokens_per_ep_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] avg=32768.0000 max=33096 max_rank=3 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=129979 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=44 tokens_per_ep_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] avg=32768.0000 max=33096 max_rank=5 min=32438 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65133 max_load_reduction_pct=0.663073 moved_to_replicas=97539 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=44 tokens_per_ep_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] avg=32768.0000 max=33096 max_rank=5 min=32438 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65133 max_load_reduction_pct=0.663073 moved_to_replicas=97539 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=41 tokens_per_ep_rank=[32511, 32768, 32768, 33096, 33096, 32328, 32481, 33096] avg=32768.0000 max=33096 max_rank=3 min=32328 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=96947 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=46 tokens_per_ep_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] avg=32768.0000 max=33096 max_rank=6 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97902 max_load_reduction_pct=0.747355 moved_to_replicas=97902 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=45 tokens_per_ep_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] avg=32768.0000 max=33096 max_rank=3 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=129979 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=46 tokens_per_ep_rank=[0, 31, 34, 32747, 32763, 32789, 130998, 32782] avg=32768.0000 max=130998 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=46 tokens_per_ep_rank=[0, 31, 34, 32747, 32763, 32789, 130998, 32782] avg=32768.0000 max=130998 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=46 tokens_per_ep_rank=[0, 31, 34, 32747, 32763, 32789, 130998, 32782] avg=32768.0000 max=130998 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=45 tokens_per_ep_rank=[42, 32768, 4, 65487, 98294, 65486, 21, 42] avg=32768.0000 max=98294 max_rank=4 min=4 min_rank=2 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=45 tokens_per_ep_rank=[42, 32768, 4, 65487, 98294, 65486, 21, 42] avg=32768.0000 max=98294 max_rank=4 min=4 min_rank=2 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 21, 47376, 32768, 65501, 32743, 18177] avg=32768.0000 max=65501 max_rank=5 min=21 min_rank=2 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=42 tokens_per_ep_rank=[32770, 65515, 29101, 65514, 42, 32773, 3683, 32746] avg=32768.0000 max=65515 max_rank=1 min=42 min_rank=4 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=46 tokens_per_ep_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] avg=32768.0000 max=33096 max_rank=6 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97902 max_load_reduction_pct=0.747355 moved_to_replicas=97902 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=46 tokens_per_ep_rank=[0, 31, 34, 32747, 32763, 32789, 130998, 32782] avg=32768.0000 max=130998 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=46 tokens_per_ep_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] avg=32768.0000 max=33096 max_rank=6 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97902 max_load_reduction_pct=0.747355 moved_to_replicas=97902 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=46 tokens_per_ep_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] avg=32768.0000 max=33096 max_rank=6 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97902 max_load_reduction_pct=0.747355 moved_to_replicas=97902 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=45 tokens_per_ep_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] avg=32768.0000 max=33096 max_rank=3 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=129979 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=45 tokens_per_ep_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] avg=32768.0000 max=33096 max_rank=3 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=129979 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=46685 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=42 tokens_per_ep_rank=[32770, 33096, 32106, 33096, 32461, 32773, 33096, 32746] avg=32768.0000 max=33096 max_rank=1 min=32106 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=64837 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=46 tokens_per_ep_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] avg=32768.0000 max=33096 max_rank=6 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97902 max_load_reduction_pct=0.747355 moved_to_replicas=97902 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 21, 47376, 32768, 65501, 32743, 18177] avg=32768.0000 max=65501 max_rank=5 min=21 min_rank=2 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 21, 47376, 32768, 65501, 32743, 18177] avg=32768.0000 max=65501 max_rank=5 min=21 min_rank=2 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 21, 47376, 32768, 65501, 32743, 18177] avg=32768.0000 max=65501 max_rank=5 min=21 min_rank=2 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=46 tokens_per_ep_rank=[0, 31, 34, 32747, 32763, 32789, 130998, 32782] avg=32768.0000 max=130998 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=46 tokens_per_ep_rank=[0, 31, 34, 32747, 32763, 32789, 130998, 32782] avg=32768.0000 max=130998 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=48 tokens_per_ep_rank=[21, 32761, 65109, 32747, 21, 65811, 65513, 161] avg=32768.0000 max=65811 max_rank=5 min=21 min_rank=0 imbalance_ratio=2.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=46685 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=46685 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=43 tokens_per_ep_rank=[64929, 79081, 21, 42, 43, 95995, 0, 22033] avg=32768.0000 max=95995 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.929535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 21, 47376, 32768, 65501, 32743, 18177] avg=32768.0000 max=65501 max_rank=5 min=21 min_rank=2 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=46685 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=46 tokens_per_ep_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] avg=32768.0000 max=33096 max_rank=6 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97902 max_load_reduction_pct=0.747355 moved_to_replicas=97902 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=46 tokens_per_ep_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] avg=32768.0000 max=33096 max_rank=6 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97902 max_load_reduction_pct=0.747355 moved_to_replicas=97902 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=48 tokens_per_ep_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] avg=32768.0000 max=33096 max_rank=2 min=32174 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.497105 moved_to_replicas=97145 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=46685 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=43 tokens_per_ep_rank=[33096, 33096, 31130, 32788, 33096, 33096, 32746, 33096] avg=32768.0000 max=33096 max_rank=0 min=31130 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=62899 max_load_reduction_pct=0.655232 moved_to_replicas=140717 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=48 tokens_per_ep_rank=[21, 32761, 65109, 32747, 21, 65811, 65513, 161] avg=32768.0000 max=65811 max_rank=5 min=21 min_rank=0 imbalance_ratio=2.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=48 tokens_per_ep_rank=[21, 32761, 65109, 32747, 21, 65811, 65513, 161] avg=32768.0000 max=65811 max_rank=5 min=21 min_rank=0 imbalance_ratio=2.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 21, 47376, 32768, 65501, 32743, 18177] avg=32768.0000 max=65501 max_rank=5 min=21 min_rank=2 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=48 tokens_per_ep_rank=[21, 32761, 65109, 32747, 21, 65811, 65513, 161] avg=32768.0000 max=65811 max_rank=5 min=21 min_rank=0 imbalance_ratio=2.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 21, 47376, 32768, 65501, 32743, 18177] avg=32768.0000 max=65501 max_rank=5 min=21 min_rank=2 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=48 tokens_per_ep_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] avg=32768.0000 max=33096 max_rank=2 min=32174 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.497105 moved_to_replicas=97145 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=48 tokens_per_ep_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] avg=32768.0000 max=33096 max_rank=2 min=32174 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.497105 moved_to_replicas=97145 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=48 tokens_per_ep_rank=[21, 32761, 65109, 32747, 21, 65811, 65513, 161] avg=32768.0000 max=65811 max_rank=5 min=21 min_rank=0 imbalance_ratio=2.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=44 tokens_per_ep_rank=[51, 42, 32747, 32780, 32724, 65502, 98229, 69] avg=32768.0000 max=98229 max_rank=6 min=42 min_rank=1 imbalance_ratio=2.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=48 tokens_per_ep_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] avg=32768.0000 max=33096 max_rank=2 min=32174 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.497105 moved_to_replicas=97145 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=46685 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=46685 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=48 tokens_per_ep_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] avg=32768.0000 max=33096 max_rank=2 min=32174 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.497105 moved_to_replicas=97145 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=44 tokens_per_ep_rank=[32438, 32788, 32747, 32780, 32724, 33096, 33096, 32475] avg=32768.0000 max=33096 max_rank=5 min=32438 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65133 max_load_reduction_pct=0.663073 moved_to_replicas=97539 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=48 tokens_per_ep_rank=[21, 32761, 65109, 32747, 21, 65811, 65513, 161] avg=32768.0000 max=65811 max_rank=5 min=21 min_rank=0 imbalance_ratio=2.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=48 tokens_per_ep_rank=[21, 32761, 65109, 32747, 21, 65811, 65513, 161] avg=32768.0000 max=65811 max_rank=5 min=21 min_rank=0 imbalance_ratio=2.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=45 tokens_per_ep_rank=[42, 32768, 4, 65487, 98294, 65486, 21, 42] avg=32768.0000 max=98294 max_rank=4 min=4 min_rank=2 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=48 tokens_per_ep_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] avg=32768.0000 max=33096 max_rank=2 min=32174 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.497105 moved_to_replicas=97145 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=48 tokens_per_ep_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] avg=32768.0000 max=33096 max_rank=2 min=32174 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.497105 moved_to_replicas=97145 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=45 tokens_per_ep_rank=[32433, 32768, 32750, 33096, 33096, 33096, 32473, 32432] avg=32768.0000 max=33096 max_rank=3 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=129979 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=46 tokens_per_ep_rank=[0, 31, 34, 32747, 32763, 32789, 130998, 32782] avg=32768.0000 max=130998 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=46 tokens_per_ep_rank=[32746, 32777, 32444, 32747, 32763, 32789, 33096, 32782] avg=32768.0000 max=33096 max_rank=6 min=32444 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97902 max_load_reduction_pct=0.747355 moved_to_replicas=97902 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 21, 47376, 32768, 65501, 32743, 18177] avg=32768.0000 max=65501 max_rank=5 min=21 min_rank=2 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=47 tokens_per_ep_rank=[32789, 32769, 32426, 33095, 32768, 33096, 32743, 32458] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=46685 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=48 tokens_per_ep_rank=[21, 32761, 65109, 32747, 21, 65811, 65513, 161] avg=32768.0000 max=65811 max_rank=5 min=21 min_rank=0 imbalance_ratio=2.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=48 tokens_per_ep_rank=[32736, 32761, 33096, 32747, 32438, 33096, 33096, 32174] avg=32768.0000 max=33096 max_rank=2 min=32174 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.497105 moved_to_replicas=97145 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=48 total_expected_recv=1572329 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377088 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=48 total_expected_recv=1573490 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377344 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=48 total_expected_recv=1572101 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377600 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=48 total_expected_recv=1573992 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375296 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=48 total_expected_recv=1574095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375808 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=48 total_expected_recv=1573193 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377728 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=48 total_expected_recv=1568118 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=382592 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=48 total_expected_recv=1575594 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374784 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:47:03] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 712.4 | throughput per GPU (TFLOP/s/GPU): 125.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.110622E+00 | loss scale: 1.0 | grad norm: 3.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=31682 expected_overflow=0 exact_recv_per_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=31885 expected_overflow=0 exact_recv_per_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=31682 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=31885 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=31536 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=30783 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=31536 actual_valid_recv=32256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=30783 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32348 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32348 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=31554 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=31554 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=30906 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=30906 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=30993 expected_overflow=0 exact_recv_per_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=30993 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=31761 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=31761 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=31534 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=31534 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32131 expected_overflow=0 exact_recv_per_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32131 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32263 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32344 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32263 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32344 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32228 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32228 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32122 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32122 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32408 expected_overflow=0 exact_recv_per_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32408 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32389 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32389 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=31749 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32145 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=31749 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32145 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32295 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32295 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=31972 expected_overflow=0 exact_recv_per_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=31972 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32277 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32277 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32629 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32629 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=31681 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=31681 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=31563 expected_overflow=0 exact_recv_per_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=31563 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=40960 expected_recv=31668 expected_overflow=0 exact_recv_per_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=40960 expected_recv=31668 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=40960 expected_recv=31759 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=40960 expected_recv=31759 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=40960 expected_recv=31522 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=40960 expected_recv=31522 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=40960 expected_recv=31862 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=40960 expected_recv=31862 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32118 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32118 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32078 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32078 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32355 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32355 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32483 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=40960 expected_recv=31655 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32483 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=40960 expected_recv=31655 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32406 expected_overflow=0 exact_recv_per_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32406 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32326 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32326 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32388 expected_overflow=0 exact_recv_per_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32165 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32388 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32070 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32070 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32165 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32313 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32313 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32111 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=40960 expected_recv=31786 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32111 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=40960 expected_recv=31786 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32282 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32108 expected_overflow=0 exact_recv_per_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32108 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=40960 expected_recv=31665 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=40960 expected_recv=31665 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32559 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=40960 expected_recv=31776 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=40960 expected_recv=31776 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32168 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32168 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32411 expected_overflow=0 exact_recv_per_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32411 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=27 calls=1 budget=40960 expected_recv=31702 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=27 calls=1 budget=40960 expected_recv=31702 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32373 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32373 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32227 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32335 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32335 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32227 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32339 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32339 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32483 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32483 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=37 calls=1 budget=40960 expected_recv=31791 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=37 calls=1 budget=40960 expected_recv=31791 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32399 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32399 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32117 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32117 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32110 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32110 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32116 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32116 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=40 calls=1 budget=40960 expected_recv=31910 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=40 calls=1 budget=40960 expected_recv=31910 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=42 calls=1 budget=40960 expected_recv=31784 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=42 calls=1 budget=40960 expected_recv=31784 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32067 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32111 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32067 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32111 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=42 calls=1 budget=40960 expected_recv=31320 expected_overflow=0 exact_recv_per_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=42 calls=1 budget=40960 expected_recv=31320 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32403 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32403 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=47 calls=1 budget=40960 expected_recv=31995 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32195 expected_overflow=0 exact_recv_per_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32195 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32507 expected_overflow=0 exact_recv_per_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=47 calls=1 budget=40960 expected_recv=31995 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32507 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=44 calls=1 budget=40960 expected_recv=31997 expected_overflow=0 exact_recv_per_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=44 calls=1 budget=40960 expected_recv=31997 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32117 expected_overflow=0 exact_recv_per_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32473 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32473 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32117 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[25193, 21021, 38140, 35369, 43003, 29067, 32934, 37417] avg=32768.0000 max=43003 max_rank=4 min=21021 min_rank=1 imbalance_ratio=1.312347 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32801, 30930, 33097, 33096, 33094, 33096, 32934, 33096] avg=32768.0000 max=33097 max_rank=2 min=30930 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9906 max_load_reduction_pct=0.230356 moved_to_replicas=21545 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[55009, 41693, 44, 25239, 17702, 15615, 91453, 15389] avg=32768.0000 max=91453 max_rank=6 min=44 min_rank=2 imbalance_ratio=2.790924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32784, 33096, 33095, 30783, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=30783 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=58356 max_load_reduction_pct=0.638098 moved_to_replicas=88867 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[13285, 60307, 13, 4748, 94912, 29394, 35256, 24229] avg=32768.0000 max=94912 max_rank=4 min=13 min_rank=2 imbalance_ratio=2.896484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[33097, 33096, 32761, 33096, 33096, 31554, 33096, 32348] avg=32768.0000 max=33097 max_rank=0 min=31554 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=61815 max_load_reduction_pct=0.651288 moved_to_replicas=91187 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[59, 32769, 32752, 45838, 32478, 35, 62756, 55457] avg=32768.0000 max=62756 max_rank=6 min=35 min_rank=5 imbalance_ratio=1.915161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[33096, 32769, 32752, 33097, 32478, 31761, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31761 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=29659 max_load_reduction_pct=0.472608 moved_to_replicas=64763 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[33307, 21, 61920, 32772, 33869, 32726, 32728, 34801] avg=32768.0000 max=61920 max_rank=2 min=21 min_rank=1 imbalance_ratio=1.889648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[33096, 31534, 33096, 32772, 33096, 32726, 32728, 33096] avg=32768.0000 max=33096 max_rank=0 min=31534 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28824 max_load_reduction_pct=0.465504 moved_to_replicas=31513 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 140, 32953, 65435, 5, 65219] avg=32768.0000 max=65435 max_rank=5 min=5 min_rank=6 imbalance_ratio=1.996918 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32772, 32641, 32979, 32263, 32953, 33096, 32344, 33096] avg=32768.0000 max=33096 max_rank=5 min=32263 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32339 max_load_reduction_pct=0.494216 moved_to_replicas=64462 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65321, 32672, 31590, 32785, 16239, 50752, 32782, 3] avg=32768.0000 max=65321 max_rank=0 min=3 min_rank=7 imbalance_ratio=1.993439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[33096, 32672, 32389, 32785, 33097, 33095, 32782, 32228] avg=32768.0000 max=33097 max_rank=4 min=32228 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.493318 moved_to_replicas=49881 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[212, 31749, 33512, 60833, 66883, 3369, 3, 65583] avg=32768.0000 max=66883 max_rank=4 min=3 min_rank=6 imbalance_ratio=2.041107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[33096, 31749, 33096, 33097, 33096, 32145, 32769, 33096] avg=32768.0000 max=33097 max_rank=3 min=31749 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33786 max_load_reduction_pct=0.505151 moved_to_replicas=94427 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32773, 65534, 160, 21875, 43515, 2, 32766, 65519] avg=32768.0000 max=65534 max_rank=1 min=2 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32773, 33096, 32583, 32295, 33095, 32440, 32766, 33096] avg=32768.0000 max=33096 max_rank=1 min=32295 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=75280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[29, 51132, 36580, 65534, 27, 32769, 32842, 43231] avg=32768.0000 max=65534 max_rank=3 min=27 min_rank=4 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[31681, 33098, 33096, 33096, 32465, 32769, 32842, 33097] avg=32768.0000 max=33098 max_rank=1 min=31681 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=64093 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[8, 65523, 32767, 65521, 65537, 32733, 45, 10] avg=32768.0000 max=65537 max_rank=4 min=8 min_rank=0 imbalance_ratio=2.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[32449, 33096, 32767, 33096, 33096, 32733, 32470, 32437] avg=32768.0000 max=33096 max_rank=1 min=32437 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=97293 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[32697, 14035, 32772, 4008, 98302, 22352, 57974, 4] avg=32768.0000 max=98302 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[32697, 33096, 32772, 33096, 33096, 31522, 33095, 32770] avg=32768.0000 max=33096 max_rank=1 min=31522 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=90084 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 65534, 10, 32736, 11084, 32769, 54468] avg=32768.0000 max=65534 max_rank=2 min=10 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32768, 32775, 33096, 32448, 32736, 32457, 32769, 33095] avg=32768.0000 max=33096 max_rank=2 min=32448 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=53810 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[64262, 65531, 32766, 13425, 32767, 20622, 32768, 3] avg=32768.0000 max=65531 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999847 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[33096, 33096, 32766, 33095, 32767, 32118, 32768, 32438] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=63601 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32782, 118733, 32753, 39, 32767, 57, 45012, 1] avg=32768.0000 max=118733 max_rank=1 min=1 min_rank=7 imbalance_ratio=3.623444 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32782, 33094, 32753, 32805, 32767, 32078, 33098, 32767] avg=32768.0000 max=33098 max_rank=6 min=32078 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=85635 max_load_reduction_pct=0.721240 moved_to_replicas=97553 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[479, 32733, 65148, 32768, 7, 32799, 32766, 65444] avg=32768.0000 max=65444 max_rank=7 min=7 min_rank=4 imbalance_ratio=1.997192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[32531, 32733, 33096, 32768, 32355, 32799, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=32355 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32348 max_load_reduction_pct=0.494285 moved_to_replicas=64400 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[2, 32769, 32767, 117779, 5897, 65517, 0, 7413] avg=32768.0000 max=117779 max_rank=3 min=0 min_rank=6 imbalance_ratio=3.594330 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[32768, 32769, 32767, 33097, 31787, 33096, 32766, 33094] avg=32768.0000 max=33097 max_rank=3 min=31787 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=84682 max_load_reduction_pct=0.718991 moved_to_replicas=117104 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[65481, 71, 3, 65696, 32772, 65351, 32768, 2] avg=32768.0000 max=65696 max_rank=3 min=2 min_rank=7 imbalance_ratio=2.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32326, 32388, 33096, 32772, 33096, 32768, 32602] avg=32768.0000 max=33096 max_rank=0 min=32326 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32600 max_load_reduction_pct=0.496225 moved_to_replicas=97240 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 65546, 860, 31951, 32765, 32770, 32829] avg=32768.0000 max=65546 max_rank=2 min=860 min_rank=3 imbalance_ratio=2.000305 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[32688, 32735, 33096, 33096, 32165, 32765, 32770, 32829] avg=32768.0000 max=33096 max_rank=2 min=32165 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32450 max_load_reduction_pct=0.495072 moved_to_replicas=32450 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49420, 34, 16291, 1, 32769, 0, 65375, 98254] avg=32768.0000 max=98254 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.998474 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[33096, 32313, 32615, 32393, 32769, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32313 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65158 max_load_reduction_pct=0.663159 moved_to_replicas=113761 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[32111, 24989, 1, 2, 97201, 40295, 3815, 63730] avg=32768.0000 max=97201 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.966339 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[32111, 33095, 32767, 31786, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31786 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64104 max_load_reduction_pct=0.659499 moved_to_replicas=101938 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[37662, 2, 0, 65653, 95955, 30102, 0, 32770] avg=32768.0000 max=95955 max_rank=4 min=0 min_rank=2 imbalance_ratio=2.928314 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[33096, 32559, 32766, 33096, 33096, 31665, 33096, 32770] avg=32768.0000 max=33096 max_rank=0 min=31665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62859 max_load_reduction_pct=0.655088 moved_to_replicas=99982 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[12727, 98302, 65523, 1, 78533, 7047, 0, 11] avg=32768.0000 max=98302 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[33095, 33096, 33096, 32441, 33097, 31776, 32766, 32777] avg=32768.0000 max=33097 max_rank=4 min=31776 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=65205 max_load_reduction_pct=0.663313 moved_to_replicas=143070 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[1, 32700, 2, 32805, 32808, 32767, 131059, 2] avg=32768.0000 max=131059 max_rank=6 min=1 min_rank=0 imbalance_ratio=3.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[32767, 32700, 32768, 32805, 32808, 32767, 33096, 32433] avg=32768.0000 max=33096 max_rank=6 min=32433 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97963 max_load_reduction_pct=0.747473 moved_to_replicas=97963 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 73073, 58051, 0, 1, 32768, 32766] avg=32768.0000 max=73073 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.230011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32720, 33097, 33094, 32766, 32168, 32768, 32766] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39976 max_load_reduction_pct=0.547069 moved_to_replicas=64932 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 65535, 36, 32766, 32743, 32714] avg=32768.0000 max=65535 max_rank=3 min=36 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=26 tokens_per_ep_rank=[32737, 32846, 32767, 33096, 32475, 32766, 32743, 32714] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=32439 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=27 tokens_per_ep_rank=[32771, 0, 32783, 3, 98293, 32759, 65534, 1] avg=32768.0000 max=98293 max_rank=4 min=0 min_rank=1 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=27 tokens_per_ep_rank=[32771, 32766, 32783, 32441, 33096, 32759, 33096, 32432] avg=32768.0000 max=33096 max_rank=4 min=32432 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=97635 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33344, 32935, 33159, 64486, 1, 32770] avg=32768.0000 max=64486 max_rank=5 min=1 min_rank=6 imbalance_ratio=1.967957 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=28 tokens_per_ep_rank=[32767, 32682, 33096, 32935, 33096, 33096, 31702, 32770] avg=32768.0000 max=33096 max_rank=2 min=31702 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31390 max_load_reduction_pct=0.486772 moved_to_replicas=31701 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=29 tokens_per_ep_rank=[2, 32818, 70, 32781, 32766, 65533, 65448, 32726] avg=32768.0000 max=65533 max_rank=5 min=2 min_rank=0 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=29 tokens_per_ep_rank=[32439, 32818, 32422, 32781, 32766, 33096, 33096, 32726] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=64789 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=30 tokens_per_ep_rank=[65535, 65459, 1, 32769, 10, 32766, 32767, 32837] avg=32768.0000 max=65535 max_rank=0 min=1 min_rank=2 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=30 tokens_per_ep_rank=[33096, 33096, 32440, 32769, 32373, 32766, 32767, 32837] avg=32768.0000 max=33096 max_rank=0 min=32373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64802 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=31 tokens_per_ep_rank=[3, 64983, 32768, 32768, 32774, 32767, 32227, 33854] avg=32768.0000 max=64983 max_rank=1 min=3 min_rank=0 imbalance_ratio=1.983124 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=31 tokens_per_ep_rank=[32648, 33096, 32768, 32768, 32774, 32767, 32227, 33096] avg=32768.0000 max=33096 max_rank=1 min=32227 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=32645 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=32 tokens_per_ep_rank=[32665, 1, 32763, 98276, 65539, 87, 32767, 46] avg=32768.0000 max=98276 max_rank=3 min=1 min_rank=1 imbalance_ratio=2.999146 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=32 tokens_per_ep_rank=[32665, 32767, 32763, 33096, 33096, 32530, 32767, 32460] avg=32768.0000 max=33096 max_rank=3 min=32460 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65180 max_load_reduction_pct=0.663234 moved_to_replicas=97623 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=33 tokens_per_ep_rank=[65509, 19, 65532, 1, 32767, 0, 98304, 12] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32432, 33096, 32443, 32767, 32766, 33096, 32448] avg=32768.0000 max=33096 max_rank=0 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=130057 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=34 tokens_per_ep_rank=[32767, 2, 33135, 32402, 32770, 32767, 32767, 65534] avg=32768.0000 max=65534 max_rank=7 min=2 min_rank=1 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=34 tokens_per_ep_rank=[32767, 32440, 33096, 32441, 32770, 32767, 32767, 33096] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32477 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=35 tokens_per_ep_rank=[3, 1, 32770, 98519, 32747, 32672, 65432, 0] avg=32768.0000 max=98519 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=35 tokens_per_ep_rank=[32339, 32658, 32770, 33096, 32747, 32672, 33096, 32766] avg=32768.0000 max=33096 max_rank=3 min=32339 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65423 max_load_reduction_pct=0.664065 moved_to_replicas=97759 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=36 tokens_per_ep_rank=[98297, 98300, 0, 1, 32768, 8, 32768, 2] avg=32768.0000 max=98300 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=36 tokens_per_ep_rank=[33096, 33096, 32766, 32439, 32768, 32443, 32768, 32768] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=130405 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 2, 32767, 44, 0, 98259, 65535] avg=32768.0000 max=98259 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.998627 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=37 tokens_per_ep_rank=[32768, 32769, 32399, 32767, 32483, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32399 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65163 max_load_reduction_pct=0.663176 moved_to_replicas=97602 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=38 tokens_per_ep_rank=[1647, 0, 128771, 1, 65518, 32767, 675, 32765] avg=32768.0000 max=128771 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.929779 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=38 tokens_per_ep_rank=[33096, 32766, 33096, 32767, 33096, 32767, 31791, 32765] avg=32768.0000 max=33096 max_rank=0 min=31791 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95675 max_load_reduction_pct=0.742986 moved_to_replicas=128097 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=39 tokens_per_ep_rank=[0, 31401, 32767, 2, 32768, 65536, 98298, 1372] avg=32768.0000 max=98298 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=39 tokens_per_ep_rank=[32766, 32117, 32767, 32438, 32768, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32117 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97642 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=40 tokens_per_ep_rank=[65535, 32768, 0, 32769, 32769, 32767, 65536, 0] avg=32768.0000 max=65536 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=40 tokens_per_ep_rank=[33096, 32768, 32440, 32769, 32769, 32767, 33096, 32439] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64879 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=41 tokens_per_ep_rank=[23, 1, 87479, 0, 32688, 65466, 11631, 64856] avg=32768.0000 max=87479 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.669647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=41 tokens_per_ep_rank=[32393, 31910, 33097, 32766, 32688, 33096, 33098, 33096] avg=32768.0000 max=33098 max_rank=6 min=31910 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=54381 max_load_reduction_pct=0.621646 moved_to_replicas=118513 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 10598, 32769, 52, 76145, 44281, 32764] avg=32768.0000 max=76145 max_rank=5 min=52 min_rank=4 imbalance_ratio=2.323761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=42 tokens_per_ep_rank=[32768, 32767, 32067, 32769, 32818, 33097, 33094, 32764] avg=32768.0000 max=33097 max_rank=5 min=32067 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=43048 max_load_reduction_pct=0.565342 moved_to_replicas=54234 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=43 tokens_per_ep_rank=[51225, 32768, 1, 31670, 2, 98301, 42, 48135] avg=32768.0000 max=98301 max_rank=5 min=1 min_rank=2 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=43 tokens_per_ep_rank=[33096, 32768, 32767, 31784, 32441, 33096, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31784 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65204 max_load_reduction_pct=0.663310 moved_to_replicas=98373 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=44 tokens_per_ep_rank=[32768, 2, 32768, 41803, 56426, 65535, 73, 32769] avg=32768.0000 max=65535 max_rank=5 min=2 min_rank=1 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=44 tokens_per_ep_rank=[32768, 32441, 32768, 33094, 33097, 33096, 32111, 32769] avg=32768.0000 max=33097 max_rank=4 min=32111 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64476 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=45 tokens_per_ep_rank=[32769, 65476, 32768, 1, 65535, 65528, 0, 67] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=45 tokens_per_ep_rank=[32769, 33096, 32768, 32433, 33096, 33096, 32439, 32447] avg=32768.0000 max=33096 max_rank=1 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97251 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=46 tokens_per_ep_rank=[31766, 2, 6, 32768, 1, 66532, 131069, 0] avg=32768.0000 max=131069 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=46 tokens_per_ep_rank=[32436, 32443, 32772, 32768, 32767, 33096, 33096, 32766] avg=32768.0000 max=33096 max_rank=5 min=32436 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97973 max_load_reduction_pct=0.747492 moved_to_replicas=131409 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=47 tokens_per_ep_rank=[1, 2, 32768, 98301, 32768, 65536, 32768, 0] avg=32768.0000 max=98301 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=47 tokens_per_ep_rank=[32440, 32442, 32768, 33096, 32768, 33096, 32768, 32766] avg=32768.0000 max=33096 max_rank=3 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=97645 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=48 tokens_per_ep_rank=[31292, 32767, 65535, 1360, 2, 98303, 32, 32853] avg=32768.0000 max=98303 max_rank=5 min=2 min_rank=4 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=48 tokens_per_ep_rank=[31995, 32767, 33096, 33096, 32768, 33096, 32473, 32853] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97646 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=48 total_expected_recv=1575478 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375680 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=48 total_expected_recv=1566961 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=384640 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=48 total_expected_recv=1573222 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378368 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=48 total_expected_recv=1573757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379008 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=48 total_expected_recv=1575025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376192 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=48 total_expected_recv=1575019 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377344 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=48 total_expected_recv=1574815 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378112 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=48 total_expected_recv=1568635 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=383104 peak_demand=33098 required_capacity=1.010071 - [2026-08-26 15:01:25] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 1091.5 | throughput per GPU (TFLOP/s/GPU): 81.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.861445E+00 | loss scale: 1.0 | grad norm: 3.114 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=30944 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34688 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=30944 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[23309, 23187, 34078, 43554, 43697, 25633, 29379, 39307] avg=32768.0000 max=43697 max_rank=4 min=23187 min_rank=1 imbalance_ratio=1.333527 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[23309, 23187, 34078, 43554, 43697, 25633, 29379, 39307] avg=32768.0000 max=43697 max_rank=4 min=23187 min_rank=1 imbalance_ratio=1.333527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[23309, 23187, 34078, 43554, 43697, 25633, 29379, 39307] avg=32768.0000 max=43697 max_rank=4 min=23187 min_rank=1 imbalance_ratio=1.333527 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[23309, 23187, 34078, 43554, 43697, 25633, 29379, 39307] avg=32768.0000 max=43697 max_rank=4 min=23187 min_rank=1 imbalance_ratio=1.333527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] avg=32768.0000 max=33097 max_rank=2 min=31682 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=10600 max_load_reduction_pct=0.242580 moved_to_replicas=28252 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] avg=32768.0000 max=33097 max_rank=2 min=31682 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=10600 max_load_reduction_pct=0.242580 moved_to_replicas=28252 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] avg=32768.0000 max=33097 max_rank=2 min=31682 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=10600 max_load_reduction_pct=0.242580 moved_to_replicas=28252 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] avg=32768.0000 max=33097 max_rank=2 min=31682 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=10600 max_load_reduction_pct=0.242580 moved_to_replicas=28252 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[23309, 23187, 34078, 43554, 43697, 25633, 29379, 39307] avg=32768.0000 max=43697 max_rank=4 min=23187 min_rank=1 imbalance_ratio=1.333527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[23309, 23187, 34078, 43554, 43697, 25633, 29379, 39307] avg=32768.0000 max=43697 max_rank=4 min=23187 min_rank=1 imbalance_ratio=1.333527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[80089, 49686, 29717, 34565, 245, 32668, 33446, 1728] avg=32768.0000 max=80089 max_rank=0 min=245 min_rank=4 imbalance_ratio=2.444122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[80089, 49686, 29717, 34565, 245, 32668, 33446, 1728] avg=32768.0000 max=80089 max_rank=0 min=245 min_rank=4 imbalance_ratio=2.444122 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[80089, 49686, 29717, 34565, 245, 32668, 33446, 1728] avg=32768.0000 max=80089 max_rank=0 min=245 min_rank=4 imbalance_ratio=2.444122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[23309, 23187, 34078, 43554, 43697, 25633, 29379, 39307] avg=32768.0000 max=43697 max_rank=4 min=23187 min_rank=1 imbalance_ratio=1.333527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] avg=32768.0000 max=33097 max_rank=2 min=31682 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=10600 max_load_reduction_pct=0.242580 moved_to_replicas=28252 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] avg=32768.0000 max=33097 max_rank=2 min=31682 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=10600 max_load_reduction_pct=0.242580 moved_to_replicas=28252 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] avg=32768.0000 max=33097 max_rank=1 min=31536 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46992 max_load_reduction_pct=0.586747 moved_to_replicas=65402 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[80089, 49686, 29717, 34565, 245, 32668, 33446, 1728] avg=32768.0000 max=80089 max_rank=0 min=245 min_rank=4 imbalance_ratio=2.444122 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] avg=32768.0000 max=33097 max_rank=1 min=31536 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46992 max_load_reduction_pct=0.586747 moved_to_replicas=65402 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] avg=32768.0000 max=33097 max_rank=1 min=31536 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46992 max_load_reduction_pct=0.586747 moved_to_replicas=65402 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] avg=32768.0000 max=33097 max_rank=2 min=31682 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=10600 max_load_reduction_pct=0.242580 moved_to_replicas=28252 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] avg=32768.0000 max=33097 max_rank=1 min=31536 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46992 max_load_reduction_pct=0.586747 moved_to_replicas=65402 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[80089, 49686, 29717, 34565, 245, 32668, 33446, 1728] avg=32768.0000 max=80089 max_rank=0 min=245 min_rank=4 imbalance_ratio=2.444122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[81512, 20631, 15216, 16296, 29985, 14983, 73337, 10184] avg=32768.0000 max=81512 max_rank=0 min=10184 min_rank=7 imbalance_ratio=2.487549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[80089, 49686, 29717, 34565, 245, 32668, 33446, 1728] avg=32768.0000 max=80089 max_rank=0 min=245 min_rank=4 imbalance_ratio=2.444122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[23309, 23187, 34078, 43554, 43697, 25633, 29379, 39307] avg=32768.0000 max=43697 max_rank=4 min=23187 min_rank=1 imbalance_ratio=1.333527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[81512, 20631, 15216, 16296, 29985, 14983, 73337, 10184] avg=32768.0000 max=81512 max_rank=0 min=10184 min_rank=7 imbalance_ratio=2.487549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[81512, 20631, 15216, 16296, 29985, 14983, 73337, 10184] avg=32768.0000 max=81512 max_rank=0 min=10184 min_rank=7 imbalance_ratio=2.487549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] avg=32768.0000 max=33097 max_rank=1 min=31536 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46992 max_load_reduction_pct=0.586747 moved_to_replicas=65402 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30906 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48415 max_load_reduction_pct=0.593962 moved_to_replicas=88657 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[80089, 49686, 29717, 34565, 245, 32668, 33446, 1728] avg=32768.0000 max=80089 max_rank=0 min=245 min_rank=4 imbalance_ratio=2.444122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] avg=32768.0000 max=33097 max_rank=1 min=31536 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46992 max_load_reduction_pct=0.586747 moved_to_replicas=65402 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[81512, 20631, 15216, 16296, 29985, 14983, 73337, 10184] avg=32768.0000 max=81512 max_rank=0 min=10184 min_rank=7 imbalance_ratio=2.487549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30906 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48415 max_load_reduction_pct=0.593962 moved_to_replicas=88657 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30906 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48415 max_load_reduction_pct=0.593962 moved_to_replicas=88657 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[33096, 31885, 33097, 33096, 33096, 33097, 31682, 33095] avg=32768.0000 max=33097 max_rank=2 min=31682 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=10600 max_load_reduction_pct=0.242580 moved_to_replicas=28252 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] avg=32768.0000 max=33097 max_rank=1 min=31536 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46992 max_load_reduction_pct=0.586747 moved_to_replicas=65402 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30906 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48415 max_load_reduction_pct=0.593962 moved_to_replicas=88657 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[838, 28119, 34085, 34685, 32781, 65481, 33265, 32890] avg=32768.0000 max=65481 max_rank=5 min=838 min_rank=0 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[81512, 20631, 15216, 16296, 29985, 14983, 73337, 10184] avg=32768.0000 max=81512 max_rank=0 min=10184 min_rank=7 imbalance_ratio=2.487549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[81512, 20631, 15216, 16296, 29985, 14983, 73337, 10184] avg=32768.0000 max=81512 max_rank=0 min=10184 min_rank=7 imbalance_ratio=2.487549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[838, 28119, 34085, 34685, 32781, 65481, 33265, 32890] avg=32768.0000 max=65481 max_rank=5 min=838 min_rank=0 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[838, 28119, 34085, 34685, 32781, 65481, 33265, 32890] avg=32768.0000 max=65481 max_rank=5 min=838 min_rank=0 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[80089, 49686, 29717, 34565, 245, 32668, 33446, 1728] avg=32768.0000 max=80089 max_rank=0 min=245 min_rank=4 imbalance_ratio=2.444122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] avg=32768.0000 max=33096 max_rank=0 min=30993 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=35132 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30906 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48415 max_load_reduction_pct=0.593962 moved_to_replicas=88657 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30906 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48415 max_load_reduction_pct=0.593962 moved_to_replicas=88657 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[838, 28119, 34085, 34685, 32781, 65481, 33265, 32890] avg=32768.0000 max=65481 max_rank=5 min=838 min_rank=0 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[81512, 20631, 15216, 16296, 29985, 14983, 73337, 10184] avg=32768.0000 max=81512 max_rank=0 min=10184 min_rank=7 imbalance_ratio=2.487549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] avg=32768.0000 max=33096 max_rank=0 min=30993 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=35132 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] avg=32768.0000 max=33096 max_rank=0 min=30993 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=35132 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33095, 33097, 31536, 33096, 32733, 32668, 33096, 32823] avg=32768.0000 max=33097 max_rank=1 min=31536 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46992 max_load_reduction_pct=0.586747 moved_to_replicas=65402 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] avg=32768.0000 max=33096 max_rank=0 min=30993 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=35132 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30906 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48415 max_load_reduction_pct=0.593962 moved_to_replicas=88657 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[8, 44880, 32843, 97927, 53684, 108, 32637, 57] avg=32768.0000 max=97927 max_rank=3 min=8 min_rank=0 imbalance_ratio=2.988495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[838, 28119, 34085, 34685, 32781, 65481, 33265, 32890] avg=32768.0000 max=65481 max_rank=5 min=838 min_rank=0 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[838, 28119, 34085, 34685, 32781, 65481, 33265, 32890] avg=32768.0000 max=65481 max_rank=5 min=838 min_rank=0 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[8, 44880, 32843, 97927, 53684, 108, 32637, 57] avg=32768.0000 max=97927 max_rank=3 min=8 min_rank=0 imbalance_ratio=2.988495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[8, 44880, 32843, 97927, 53684, 108, 32637, 57] avg=32768.0000 max=97927 max_rank=3 min=8 min_rank=0 imbalance_ratio=2.988495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[81512, 20631, 15216, 16296, 29985, 14983, 73337, 10184] avg=32768.0000 max=81512 max_rank=0 min=10184 min_rank=7 imbalance_ratio=2.487549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64831 max_load_reduction_pct=0.662034 moved_to_replicas=97203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] avg=32768.0000 max=33096 max_rank=0 min=30993 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=35132 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] avg=32768.0000 max=33096 max_rank=0 min=30993 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=35132 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[8, 44880, 32843, 97927, 53684, 108, 32637, 57] avg=32768.0000 max=97927 max_rank=3 min=8 min_rank=0 imbalance_ratio=2.988495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[838, 28119, 34085, 34685, 32781, 65481, 33265, 32890] avg=32768.0000 max=65481 max_rank=5 min=838 min_rank=0 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64831 max_load_reduction_pct=0.662034 moved_to_replicas=97203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64831 max_load_reduction_pct=0.662034 moved_to_replicas=97203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[33096, 33095, 30906, 33097, 32662, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30906 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48415 max_load_reduction_pct=0.593962 moved_to_replicas=88657 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64831 max_load_reduction_pct=0.662034 moved_to_replicas=97203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] avg=32768.0000 max=33096 max_rank=0 min=30993 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=35132 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[34334, 32725, 29, 32760, 32782, 63951, 7, 65556] avg=32768.0000 max=65556 max_rank=7 min=7 min_rank=6 imbalance_ratio=2.000610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[8, 44880, 32843, 97927, 53684, 108, 32637, 57] avg=32768.0000 max=97927 max_rank=3 min=8 min_rank=0 imbalance_ratio=2.988495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[8, 44880, 32843, 97927, 53684, 108, 32637, 57] avg=32768.0000 max=97927 max_rank=3 min=8 min_rank=0 imbalance_ratio=2.988495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[34334, 32725, 29, 32760, 32782, 63951, 7, 65556] avg=32768.0000 max=65556 max_rank=7 min=7 min_rank=6 imbalance_ratio=2.000610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[34334, 32725, 29, 32760, 32782, 63951, 7, 65556] avg=32768.0000 max=65556 max_rank=7 min=7 min_rank=6 imbalance_ratio=2.000610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] avg=32768.0000 max=33096 max_rank=0 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32460 max_load_reduction_pct=0.495149 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64831 max_load_reduction_pct=0.662034 moved_to_replicas=97203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[838, 28119, 34085, 34685, 32781, 65481, 33265, 32890] avg=32768.0000 max=65481 max_rank=5 min=838 min_rank=0 imbalance_ratio=1.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64831 max_load_reduction_pct=0.662034 moved_to_replicas=97203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[34334, 32725, 29, 32760, 32782, 63951, 7, 65556] avg=32768.0000 max=65556 max_rank=7 min=7 min_rank=6 imbalance_ratio=2.000610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] avg=32768.0000 max=33096 max_rank=0 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32460 max_load_reduction_pct=0.495149 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[8, 44880, 32843, 97927, 53684, 108, 32637, 57] avg=32768.0000 max=97927 max_rank=3 min=8 min_rank=0 imbalance_ratio=2.988495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] avg=32768.0000 max=33096 max_rank=0 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32460 max_load_reduction_pct=0.495149 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[33096, 30993, 33096, 33096, 32781, 33096, 33096, 32890] avg=32768.0000 max=33096 max_rank=0 min=30993 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32385 max_load_reduction_pct=0.494571 moved_to_replicas=35132 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] avg=32768.0000 max=33096 max_rank=0 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32460 max_load_reduction_pct=0.495149 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32167, 33337, 32840, 32766, 32767, 98161, 16, 90] avg=32768.0000 max=98161 max_rank=5 min=16 min_rank=6 imbalance_ratio=2.995636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[34334, 32725, 29, 32760, 32782, 63951, 7, 65556] avg=32768.0000 max=65556 max_rank=7 min=7 min_rank=6 imbalance_ratio=2.000610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[34334, 32725, 29, 32760, 32782, 63951, 7, 65556] avg=32768.0000 max=65556 max_rank=7 min=7 min_rank=6 imbalance_ratio=2.000610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64831 max_load_reduction_pct=0.662034 moved_to_replicas=97203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32167, 33337, 32840, 32766, 32767, 98161, 16, 90] avg=32768.0000 max=98161 max_rank=5 min=16 min_rank=6 imbalance_ratio=2.995636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32167, 33337, 32840, 32766, 32767, 98161, 16, 90] avg=32768.0000 max=98161 max_rank=5 min=16 min_rank=6 imbalance_ratio=2.995636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] avg=32768.0000 max=33096 max_rank=1 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65065 max_load_reduction_pct=0.662840 moved_to_replicas=65306 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] avg=32768.0000 max=33096 max_rank=0 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32460 max_load_reduction_pct=0.495149 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] avg=32768.0000 max=33096 max_rank=0 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32460 max_load_reduction_pct=0.495149 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[8, 44880, 32843, 97927, 53684, 108, 32637, 57] avg=32768.0000 max=97927 max_rank=3 min=8 min_rank=0 imbalance_ratio=2.988495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32167, 33337, 32840, 32766, 32767, 98161, 16, 90] avg=32768.0000 max=98161 max_rank=5 min=16 min_rank=6 imbalance_ratio=2.995636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] avg=32768.0000 max=33096 max_rank=1 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65065 max_load_reduction_pct=0.662840 moved_to_replicas=65306 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] avg=32768.0000 max=33096 max_rank=1 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65065 max_load_reduction_pct=0.662840 moved_to_replicas=65306 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[34334, 32725, 29, 32760, 32782, 63951, 7, 65556] avg=32768.0000 max=65556 max_rank=7 min=7 min_rank=6 imbalance_ratio=2.000610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32765, 9, 32791, 31796, 65977, 33272, 7, 65527] avg=32768.0000 max=65977 max_rank=4 min=7 min_rank=6 imbalance_ratio=2.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] avg=32768.0000 max=33096 max_rank=1 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65065 max_load_reduction_pct=0.662840 moved_to_replicas=65306 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32765, 33095, 32843, 33096, 33096, 32481, 32637, 32131] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64831 max_load_reduction_pct=0.662034 moved_to_replicas=97203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32167, 33337, 32840, 32766, 32767, 98161, 16, 90] avg=32768.0000 max=98161 max_rank=5 min=16 min_rank=6 imbalance_ratio=2.995636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] avg=32768.0000 max=33096 max_rank=0 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32460 max_load_reduction_pct=0.495149 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32167, 33337, 32840, 32766, 32767, 98161, 16, 90] avg=32768.0000 max=98161 max_rank=5 min=16 min_rank=6 imbalance_ratio=2.995636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32765, 9, 32791, 31796, 65977, 33272, 7, 65527] avg=32768.0000 max=65977 max_rank=4 min=7 min_rank=6 imbalance_ratio=2.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32765, 9, 32791, 31796, 65977, 33272, 7, 65527] avg=32768.0000 max=65977 max_rank=4 min=7 min_rank=6 imbalance_ratio=2.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=31972 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32881 max_load_reduction_pct=0.498371 moved_to_replicas=65488 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] avg=32768.0000 max=33096 max_rank=1 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65065 max_load_reduction_pct=0.662840 moved_to_replicas=65306 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] avg=32768.0000 max=33096 max_rank=1 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65065 max_load_reduction_pct=0.662840 moved_to_replicas=65306 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32765, 9, 32791, 31796, 65977, 33272, 7, 65527] avg=32768.0000 max=65977 max_rank=4 min=7 min_rank=6 imbalance_ratio=2.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[34334, 32725, 29, 32760, 32782, 63951, 7, 65556] avg=32768.0000 max=65556 max_rank=7 min=7 min_rank=6 imbalance_ratio=2.000610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=31972 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32881 max_load_reduction_pct=0.498371 moved_to_replicas=65488 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=31972 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32881 max_load_reduction_pct=0.498371 moved_to_replicas=65488 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32167, 33337, 32840, 32766, 32767, 98161, 16, 90] avg=32768.0000 max=98161 max_rank=5 min=16 min_rank=6 imbalance_ratio=2.995636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[62452, 65722, 32763, 3, 5, 32745, 2921, 65533] avg=32768.0000 max=65722 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=31972 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32881 max_load_reduction_pct=0.498371 moved_to_replicas=65488 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32725, 32122, 32760, 32782, 33096, 32467, 33096] avg=32768.0000 max=33096 max_rank=0 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32460 max_load_reduction_pct=0.495149 moved_to_replicas=64553 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32765, 9, 32791, 31796, 65977, 33272, 7, 65527] avg=32768.0000 max=65977 max_rank=4 min=7 min_rank=6 imbalance_ratio=2.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] avg=32768.0000 max=33096 max_rank=1 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65065 max_load_reduction_pct=0.662840 moved_to_replicas=65306 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32765, 9, 32791, 31796, 65977, 33272, 7, 65527] avg=32768.0000 max=65977 max_rank=4 min=7 min_rank=6 imbalance_ratio=2.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[62452, 65722, 32763, 3, 5, 32745, 2921, 65533] avg=32768.0000 max=65722 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[62452, 65722, 32763, 3, 5, 32745, 2921, 65533] avg=32768.0000 max=65722 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] avg=32768.0000 max=33096 max_rank=0 min=32277 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32626 max_load_reduction_pct=0.496424 moved_to_replicas=94419 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=31972 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32881 max_load_reduction_pct=0.498371 moved_to_replicas=65488 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[62452, 65722, 32763, 3, 5, 32745, 2921, 65533] avg=32768.0000 max=65722 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=31972 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32881 max_load_reduction_pct=0.498371 moved_to_replicas=65488 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] avg=32768.0000 max=33096 max_rank=0 min=32277 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32626 max_load_reduction_pct=0.496424 moved_to_replicas=94419 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] avg=32768.0000 max=33096 max_rank=0 min=32277 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32626 max_load_reduction_pct=0.496424 moved_to_replicas=94419 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32167, 33337, 32840, 32766, 32767, 98161, 16, 90] avg=32768.0000 max=98161 max_rank=5 min=16 min_rank=6 imbalance_ratio=2.995636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32765, 9, 32791, 31796, 65977, 33272, 7, 65527] avg=32768.0000 max=65977 max_rank=4 min=7 min_rank=6 imbalance_ratio=2.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 65517, 6, 4, 73102, 32667, 90845] avg=32768.0000 max=90845 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.772369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] avg=32768.0000 max=33096 max_rank=0 min=32277 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32626 max_load_reduction_pct=0.496424 moved_to_replicas=94419 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[62452, 65722, 32763, 3, 5, 32745, 2921, 65533] avg=32768.0000 max=65722 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=31972 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32881 max_load_reduction_pct=0.498371 moved_to_replicas=65488 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[62452, 65722, 32763, 3, 5, 32745, 2921, 65533] avg=32768.0000 max=65722 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32408, 33096, 32840, 32766, 32767, 33096, 32780, 32391] avg=32768.0000 max=33096 max_rank=1 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65065 max_load_reduction_pct=0.662840 moved_to_replicas=65306 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 65517, 6, 4, 73102, 32667, 90845] avg=32768.0000 max=90845 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.772369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 65517, 6, 4, 73102, 32667, 90845] avg=32768.0000 max=90845 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.772369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] avg=32768.0000 max=33098 max_rank=5 min=31563 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=57747 max_load_reduction_pct=0.635665 moved_to_replicas=130176 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] avg=32768.0000 max=33096 max_rank=0 min=32277 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32626 max_load_reduction_pct=0.496424 moved_to_replicas=94419 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 65517, 6, 4, 73102, 32667, 90845] avg=32768.0000 max=90845 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.772369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] avg=32768.0000 max=33096 max_rank=0 min=32277 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32626 max_load_reduction_pct=0.496424 moved_to_replicas=94419 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] avg=32768.0000 max=33098 max_rank=5 min=31563 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=57747 max_load_reduction_pct=0.635665 moved_to_replicas=130176 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] avg=32768.0000 max=33098 max_rank=5 min=31563 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=57747 max_load_reduction_pct=0.635665 moved_to_replicas=130176 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[62452, 65722, 32763, 3, 5, 32745, 2921, 65533] avg=32768.0000 max=65722 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32765, 9, 32791, 31796, 65977, 33272, 7, 65527] avg=32768.0000 max=65977 max_rank=4 min=7 min_rank=6 imbalance_ratio=2.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[28308, 31668, 71090, 65522, 32772, 32768, 4, 12] avg=32768.0000 max=71090 max_rank=2 min=4 min_rank=6 imbalance_ratio=2.169495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] avg=32768.0000 max=33098 max_rank=5 min=31563 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=57747 max_load_reduction_pct=0.635665 moved_to_replicas=130176 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 65517, 6, 4, 73102, 32667, 90845] avg=32768.0000 max=90845 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.772369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] avg=32768.0000 max=33096 max_rank=0 min=32277 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32626 max_load_reduction_pct=0.496424 moved_to_replicas=94419 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32765, 32558, 32791, 31972, 33096, 33096, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=31972 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32881 max_load_reduction_pct=0.498371 moved_to_replicas=65488 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] avg=32768.0000 max=33098 max_rank=2 min=31668 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=37992 max_load_reduction_pct=0.534421 moved_to_replicas=70420 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 65517, 6, 4, 73102, 32667, 90845] avg=32768.0000 max=90845 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.772369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[28308, 31668, 71090, 65522, 32772, 32768, 4, 12] avg=32768.0000 max=71090 max_rank=2 min=4 min_rank=6 imbalance_ratio=2.169495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[28308, 31668, 71090, 65522, 32772, 32768, 4, 12] avg=32768.0000 max=71090 max_rank=2 min=4 min_rank=6 imbalance_ratio=2.169495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] avg=32768.0000 max=33098 max_rank=5 min=31563 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=57747 max_load_reduction_pct=0.635665 moved_to_replicas=130176 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[28308, 31668, 71090, 65522, 32772, 32768, 4, 12] avg=32768.0000 max=71090 max_rank=2 min=4 min_rank=6 imbalance_ratio=2.169495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] avg=32768.0000 max=33098 max_rank=2 min=31668 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=37992 max_load_reduction_pct=0.534421 moved_to_replicas=70420 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] avg=32768.0000 max=33098 max_rank=5 min=31563 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=57747 max_load_reduction_pct=0.635665 moved_to_replicas=130176 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 65517, 6, 4, 73102, 32667, 90845] avg=32768.0000 max=90845 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.772369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] avg=32768.0000 max=33098 max_rank=2 min=31668 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=37992 max_load_reduction_pct=0.534421 moved_to_replicas=70420 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[62452, 65722, 32763, 3, 5, 32745, 2921, 65533] avg=32768.0000 max=65722 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[65531, 55891, 44341, 926, 29889, 6, 32790, 32770] avg=32768.0000 max=65531 max_rank=0 min=6 min_rank=5 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] avg=32768.0000 max=33098 max_rank=2 min=31668 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=37992 max_load_reduction_pct=0.534421 moved_to_replicas=70420 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[28308, 31668, 71090, 65522, 32772, 32768, 4, 12] avg=32768.0000 max=71090 max_rank=2 min=4 min_rank=6 imbalance_ratio=2.169495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] avg=32768.0000 max=33098 max_rank=5 min=31563 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=57747 max_load_reduction_pct=0.635665 moved_to_replicas=130176 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32434 max_load_reduction_pct=0.494941 moved_to_replicas=66475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[33096, 33096, 32763, 32629, 32442, 32745, 32277, 33096] avg=32768.0000 max=33096 max_rank=0 min=32277 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32626 max_load_reduction_pct=0.496424 moved_to_replicas=94419 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[65531, 55891, 44341, 926, 29889, 6, 32790, 32770] avg=32768.0000 max=65531 max_rank=0 min=6 min_rank=5 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[28308, 31668, 71090, 65522, 32772, 32768, 4, 12] avg=32768.0000 max=71090 max_rank=2 min=4 min_rank=6 imbalance_ratio=2.169495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[65531, 55891, 44341, 926, 29889, 6, 32790, 32770] avg=32768.0000 max=65531 max_rank=0 min=6 min_rank=5 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] avg=32768.0000 max=33098 max_rank=2 min=31668 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=37992 max_load_reduction_pct=0.534421 moved_to_replicas=70420 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[65531, 55891, 44341, 926, 29889, 6, 32790, 32770] avg=32768.0000 max=65531 max_rank=0 min=6 min_rank=5 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32434 max_load_reduction_pct=0.494941 moved_to_replicas=66475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] avg=32768.0000 max=33098 max_rank=2 min=31668 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=37992 max_load_reduction_pct=0.534421 moved_to_replicas=70420 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32434 max_load_reduction_pct=0.494941 moved_to_replicas=66475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[28308, 31668, 71090, 65522, 32772, 32768, 4, 12] avg=32768.0000 max=71090 max_rank=2 min=4 min_rank=6 imbalance_ratio=2.169495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32434 max_load_reduction_pct=0.494941 moved_to_replicas=66475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[37790, 7, 125779, 32769, 5, 32805, 8, 32981] avg=32768.0000 max=125779 max_rank=2 min=5 min_rank=4 imbalance_ratio=3.838470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 65517, 6, 4, 73102, 32667, 90845] avg=32768.0000 max=90845 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.772369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[65531, 55891, 44341, 926, 29889, 6, 32790, 32770] avg=32768.0000 max=65531 max_rank=0 min=6 min_rank=5 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] avg=32768.0000 max=33098 max_rank=2 min=31668 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=37992 max_load_reduction_pct=0.534421 moved_to_replicas=70420 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] avg=32768.0000 max=33096 max_rank=2 min=31862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=92683 max_load_reduction_pct=0.736872 moved_to_replicas=97377 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[37790, 7, 125779, 32769, 5, 32805, 8, 32981] avg=32768.0000 max=125779 max_rank=2 min=5 min_rank=4 imbalance_ratio=3.838470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32763, 31563, 33096, 33093, 32768, 33098, 32667, 33096] avg=32768.0000 max=33098 max_rank=5 min=31563 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=57747 max_load_reduction_pct=0.635665 moved_to_replicas=130176 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[37790, 7, 125779, 32769, 5, 32805, 8, 32981] avg=32768.0000 max=125779 max_rank=2 min=5 min_rank=4 imbalance_ratio=3.838470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[65531, 55891, 44341, 926, 29889, 6, 32790, 32770] avg=32768.0000 max=65531 max_rank=0 min=6 min_rank=5 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[37790, 7, 125779, 32769, 5, 32805, 8, 32981] avg=32768.0000 max=125779 max_rank=2 min=5 min_rank=4 imbalance_ratio=3.838470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32434 max_load_reduction_pct=0.494941 moved_to_replicas=66475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] avg=32768.0000 max=33096 max_rank=2 min=31862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=92683 max_load_reduction_pct=0.736872 moved_to_replicas=97377 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] avg=32768.0000 max=33096 max_rank=2 min=31862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=92683 max_load_reduction_pct=0.736872 moved_to_replicas=97377 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32434 max_load_reduction_pct=0.494941 moved_to_replicas=66475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[65531, 55891, 44341, 926, 29889, 6, 32790, 32770] avg=32768.0000 max=65531 max_rank=0 min=6 min_rank=5 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] avg=32768.0000 max=33096 max_rank=2 min=31862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=92683 max_load_reduction_pct=0.736872 moved_to_replicas=97377 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 65538, 32565, 32746, 15, 32884] avg=32768.0000 max=65538 max_rank=3 min=15 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[28308, 31668, 71090, 65522, 32772, 32768, 4, 12] avg=32768.0000 max=71090 max_rank=2 min=4 min_rank=6 imbalance_ratio=2.169495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[37790, 7, 125779, 32769, 5, 32805, 8, 32981] avg=32768.0000 max=125779 max_rank=2 min=5 min_rank=4 imbalance_ratio=3.838470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32434 max_load_reduction_pct=0.494941 moved_to_replicas=66475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] avg=32768.0000 max=33096 max_rank=3 min=32457 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=32442 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 65538, 32565, 32746, 15, 32884] avg=32768.0000 max=65538 max_rank=3 min=15 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[32878, 31668, 33098, 33096, 32772, 32768, 32768, 33096] avg=32768.0000 max=33098 max_rank=2 min=31668 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=37992 max_load_reduction_pct=0.534421 moved_to_replicas=70420 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 65538, 32565, 32746, 15, 32884] avg=32768.0000 max=65538 max_rank=3 min=15 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[37790, 7, 125779, 32769, 5, 32805, 8, 32981] avg=32768.0000 max=125779 max_rank=2 min=5 min_rank=4 imbalance_ratio=3.838470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] avg=32768.0000 max=33096 max_rank=2 min=31862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=92683 max_load_reduction_pct=0.736872 moved_to_replicas=97377 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 65538, 32565, 32746, 15, 32884] avg=32768.0000 max=65538 max_rank=3 min=15 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] avg=32768.0000 max=33096 max_rank=3 min=32457 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=32442 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] avg=32768.0000 max=33096 max_rank=3 min=32457 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=32442 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] avg=32768.0000 max=33096 max_rank=2 min=31862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=92683 max_load_reduction_pct=0.736872 moved_to_replicas=97377 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[37790, 7, 125779, 32769, 5, 32805, 8, 32981] avg=32768.0000 max=125779 max_rank=2 min=5 min_rank=4 imbalance_ratio=3.838470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[65535, 98295, 3, 32733, 1, 32768, 32765, 44] avg=32768.0000 max=98295 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] avg=32768.0000 max=33096 max_rank=3 min=32457 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=32442 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[65531, 55891, 44341, 926, 29889, 6, 32790, 32770] avg=32768.0000 max=65531 max_rank=0 min=6 min_rank=5 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 65538, 32565, 32746, 15, 32884] avg=32768.0000 max=65538 max_rank=3 min=15 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] avg=32768.0000 max=33096 max_rank=2 min=31862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=92683 max_load_reduction_pct=0.736872 moved_to_replicas=97377 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] avg=32768.0000 max=33096 max_rank=0 min=32438 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65199 max_load_reduction_pct=0.663299 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[65535, 98295, 3, 32733, 1, 32768, 32765, 44] avg=32768.0000 max=98295 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[65535, 98295, 3, 32733, 1, 32768, 32765, 44] avg=32768.0000 max=98295 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 31759, 32441, 32790, 32770] avg=32768.0000 max=33097 max_rank=1 min=31759 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32434 max_load_reduction_pct=0.494941 moved_to_replicas=66475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 65538, 32565, 32746, 15, 32884] avg=32768.0000 max=65538 max_rank=3 min=15 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[65535, 98295, 3, 32733, 1, 32768, 32765, 44] avg=32768.0000 max=98295 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] avg=32768.0000 max=33096 max_rank=3 min=32457 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=32442 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] avg=32768.0000 max=33096 max_rank=0 min=32438 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65199 max_load_reduction_pct=0.663299 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] avg=32768.0000 max=33096 max_rank=0 min=32438 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65199 max_load_reduction_pct=0.663299 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] avg=32768.0000 max=33096 max_rank=3 min=32457 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=32442 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 65538, 32565, 32746, 15, 32884] avg=32768.0000 max=65538 max_rank=3 min=15 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[65529, 0, 32765, 32773, 49, 32901, 12629, 85498] avg=32768.0000 max=85498 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.609192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] avg=32768.0000 max=33096 max_rank=0 min=32438 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65199 max_load_reduction_pct=0.663299 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[37790, 7, 125779, 32769, 5, 32805, 8, 32981] avg=32768.0000 max=125779 max_rank=2 min=5 min_rank=4 imbalance_ratio=3.838470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[65535, 98295, 3, 32733, 1, 32768, 32765, 44] avg=32768.0000 max=98295 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[65529, 0, 32765, 32773, 49, 32901, 12629, 85498] avg=32768.0000 max=85498 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.609192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] avg=32768.0000 max=33096 max_rank=3 min=32457 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=32442 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52402 max_load_reduction_pct=0.612903 moved_to_replicas=84835 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[65529, 0, 32765, 32773, 49, 32901, 12629, 85498] avg=32768.0000 max=85498 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.609192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[65535, 98295, 3, 32733, 1, 32768, 32765, 44] avg=32768.0000 max=98295 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[33094, 32769, 33096, 32769, 32768, 32805, 31862, 32981] avg=32768.0000 max=33096 max_rank=2 min=31862 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=92683 max_load_reduction_pct=0.736872 moved_to_replicas=97377 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[65529, 0, 32765, 32773, 49, 32901, 12629, 85498] avg=32768.0000 max=85498 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.609192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] avg=32768.0000 max=33096 max_rank=0 min=32438 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65199 max_load_reduction_pct=0.663299 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52402 max_load_reduction_pct=0.612903 moved_to_replicas=84835 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52402 max_load_reduction_pct=0.612903 moved_to_replicas=84835 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] avg=32768.0000 max=33096 max_rank=0 min=32438 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65199 max_load_reduction_pct=0.663299 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52402 max_load_reduction_pct=0.612903 moved_to_replicas=84835 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[65531, 11, 5, 98261, 32802, 32768, 32763, 3] avg=32768.0000 max=98261 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[65535, 98295, 3, 32733, 1, 32768, 32765, 44] avg=32768.0000 max=98295 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 65538, 32565, 32746, 15, 32884] avg=32768.0000 max=65538 max_rank=3 min=15 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[65529, 0, 32765, 32773, 49, 32901, 12629, 85498] avg=32768.0000 max=85498 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.609192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[65531, 11, 5, 98261, 32802, 32768, 32763, 3] avg=32768.0000 max=98261 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] avg=32768.0000 max=33096 max_rank=0 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65165 max_load_reduction_pct=0.663183 moved_to_replicas=97600 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[65531, 11, 5, 98261, 32802, 32768, 32763, 3] avg=32768.0000 max=98261 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] avg=32768.0000 max=33096 max_rank=0 min=32438 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65199 max_load_reduction_pct=0.663299 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[65529, 0, 32765, 32773, 49, 32901, 12629, 85498] avg=32768.0000 max=85498 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.609192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32855, 32772, 32769, 33096, 32565, 32746, 32457, 32884] avg=32768.0000 max=33096 max_rank=3 min=32457 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=32442 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52402 max_load_reduction_pct=0.612903 moved_to_replicas=84835 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[65531, 11, 5, 98261, 32802, 32768, 32763, 3] avg=32768.0000 max=98261 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] avg=32768.0000 max=33096 max_rank=0 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65165 max_load_reduction_pct=0.663183 moved_to_replicas=97600 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] avg=32768.0000 max=33096 max_rank=0 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65165 max_load_reduction_pct=0.663183 moved_to_replicas=97600 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52402 max_load_reduction_pct=0.612903 moved_to_replicas=84835 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[65529, 3, 62647, 68376, 6, 65533, 47, 3] avg=32768.0000 max=68376 max_rank=3 min=3 min_rank=1 imbalance_ratio=2.086670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] avg=32768.0000 max=33096 max_rank=0 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65165 max_load_reduction_pct=0.663183 moved_to_replicas=97600 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[65529, 0, 32765, 32773, 49, 32901, 12629, 85498] avg=32768.0000 max=85498 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.609192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[65535, 98295, 3, 32733, 1, 32768, 32765, 44] avg=32768.0000 max=98295 max_rank=1 min=1 min_rank=4 imbalance_ratio=2.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[65531, 11, 5, 98261, 32802, 32768, 32763, 3] avg=32768.0000 max=98261 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[65529, 3, 62647, 68376, 6, 65533, 47, 3] avg=32768.0000 max=68376 max_rank=3 min=3 min_rank=1 imbalance_ratio=2.086670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] avg=32768.0000 max=33096 max_rank=0 min=32070 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=35280 max_load_reduction_pct=0.515971 moved_to_replicas=129701 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[65529, 3, 62647, 68376, 6, 65533, 47, 3] avg=32768.0000 max=68376 max_rank=3 min=3 min_rank=1 imbalance_ratio=2.086670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52402 max_load_reduction_pct=0.612903 moved_to_replicas=84835 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[65531, 11, 5, 98261, 32802, 32768, 32763, 3] avg=32768.0000 max=98261 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[33096, 33096, 32438, 32733, 32765, 32768, 32765, 32483] avg=32768.0000 max=33096 max_rank=0 min=32438 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65199 max_load_reduction_pct=0.663299 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[65529, 3, 62647, 68376, 6, 65533, 47, 3] avg=32768.0000 max=68376 max_rank=3 min=3 min_rank=1 imbalance_ratio=2.086670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] avg=32768.0000 max=33096 max_rank=0 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65165 max_load_reduction_pct=0.663183 moved_to_replicas=97600 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] avg=32768.0000 max=33096 max_rank=0 min=32070 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=35280 max_load_reduction_pct=0.515971 moved_to_replicas=129701 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] avg=32768.0000 max=33096 max_rank=0 min=32070 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=35280 max_load_reduction_pct=0.515971 moved_to_replicas=129701 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] avg=32768.0000 max=33096 max_rank=0 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65165 max_load_reduction_pct=0.663183 moved_to_replicas=97600 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[32758, 65515, 5, 6, 24, 32765, 32774, 98297] avg=32768.0000 max=98297 max_rank=7 min=5 min_rank=2 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] avg=32768.0000 max=33096 max_rank=0 min=32070 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=35280 max_load_reduction_pct=0.515971 moved_to_replicas=129701 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[65531, 11, 5, 98261, 32802, 32768, 32763, 3] avg=32768.0000 max=98261 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[65529, 0, 32765, 32773, 49, 32901, 12629, 85498] avg=32768.0000 max=85498 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.609192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[65529, 3, 62647, 68376, 6, 65533, 47, 3] avg=32768.0000 max=68376 max_rank=3 min=3 min_rank=1 imbalance_ratio=2.086670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[32758, 65515, 5, 6, 24, 32765, 32774, 98297] avg=32768.0000 max=98297 max_rank=7 min=5 min_rank=2 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=97620 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[32758, 65515, 5, 6, 24, 32765, 32774, 98297] avg=32768.0000 max=98297 max_rank=7 min=5 min_rank=2 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] avg=32768.0000 max=33096 max_rank=0 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65165 max_load_reduction_pct=0.663183 moved_to_replicas=97600 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[65529, 3, 62647, 68376, 6, 65533, 47, 3] avg=32768.0000 max=68376 max_rank=3 min=3 min_rank=1 imbalance_ratio=2.086670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32764, 32765, 32773, 31655, 32901, 33096, 33094] avg=32768.0000 max=33096 max_rank=0 min=31655 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52402 max_load_reduction_pct=0.612903 moved_to_replicas=84835 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[32758, 65515, 5, 6, 24, 32765, 32774, 98297] avg=32768.0000 max=98297 max_rank=7 min=5 min_rank=2 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] avg=32768.0000 max=33096 max_rank=0 min=32070 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=35280 max_load_reduction_pct=0.515971 moved_to_replicas=129701 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=97620 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=97620 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] avg=32768.0000 max=33096 max_rank=0 min=32070 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=35280 max_load_reduction_pct=0.515971 moved_to_replicas=129701 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[6, 61975, 2, 65526, 32771, 0, 32777, 69087] avg=32768.0000 max=69087 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.108368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=97620 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[65529, 3, 62647, 68376, 6, 65533, 47, 3] avg=32768.0000 max=68376 max_rank=3 min=3 min_rank=1 imbalance_ratio=2.086670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[65531, 11, 5, 98261, 32802, 32768, 32763, 3] avg=32768.0000 max=98261 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[32758, 65515, 5, 6, 24, 32765, 32774, 98297] avg=32768.0000 max=98297 max_rank=7 min=5 min_rank=2 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[6, 61975, 2, 65526, 32771, 0, 32777, 69087] avg=32768.0000 max=69087 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.108368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] avg=32768.0000 max=33096 max_rank=1 min=32108 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35991 max_load_reduction_pct=0.520952 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[6, 61975, 2, 65526, 32771, 0, 32777, 69087] avg=32768.0000 max=69087 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.108368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] avg=32768.0000 max=33096 max_rank=0 min=32070 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=35280 max_load_reduction_pct=0.515971 moved_to_replicas=129701 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[32758, 65515, 5, 6, 24, 32765, 32774, 98297] avg=32768.0000 max=98297 max_rank=7 min=5 min_rank=2 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[6, 61975, 2, 65526, 32771, 0, 32777, 69087] avg=32768.0000 max=69087 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.108368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32446, 32406, 33096, 32802, 32768, 32763, 32767] avg=32768.0000 max=33096 max_rank=0 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65165 max_load_reduction_pct=0.663183 moved_to_replicas=97600 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] avg=32768.0000 max=33096 max_rank=1 min=32108 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35991 max_load_reduction_pct=0.520952 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=97620 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] avg=32768.0000 max=33096 max_rank=1 min=32108 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35991 max_load_reduction_pct=0.520952 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=97620 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[10, 65535, 32568, 65969, 65259, 0, 12, 32791] avg=32768.0000 max=65969 max_rank=3 min=0 min_rank=5 imbalance_ratio=2.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] avg=32768.0000 max=33096 max_rank=1 min=32108 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35991 max_load_reduction_pct=0.520952 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[32758, 65515, 5, 6, 24, 32765, 32774, 98297] avg=32768.0000 max=98297 max_rank=7 min=5 min_rank=2 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[10, 65535, 32568, 65969, 65259, 0, 12, 32791] avg=32768.0000 max=65969 max_rank=3 min=0 min_rank=5 imbalance_ratio=2.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] avg=32768.0000 max=33096 max_rank=1 min=32282 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32873 max_load_reduction_pct=0.498310 moved_to_replicas=97475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[65529, 3, 62647, 68376, 6, 65533, 47, 3] avg=32768.0000 max=68376 max_rank=3 min=3 min_rank=1 imbalance_ratio=2.086670 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[6, 61975, 2, 65526, 32771, 0, 32777, 69087] avg=32768.0000 max=69087 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.108368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[10, 65535, 32568, 65969, 65259, 0, 12, 32791] avg=32768.0000 max=65969 max_rank=3 min=0 min_rank=5 imbalance_ratio=2.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=97620 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[6, 61975, 2, 65526, 32771, 0, 32777, 69087] avg=32768.0000 max=69087 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.108368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[10, 65535, 32568, 65969, 65259, 0, 12, 32791] avg=32768.0000 max=65969 max_rank=3 min=0 min_rank=5 imbalance_ratio=2.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] avg=32768.0000 max=33096 max_rank=1 min=32282 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32873 max_load_reduction_pct=0.498310 moved_to_replicas=97475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] avg=32768.0000 max=33096 max_rank=1 min=32108 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35991 max_load_reduction_pct=0.520952 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[33096, 32767, 33096, 33096, 32443, 33096, 32480, 32070] avg=32768.0000 max=33096 max_rank=0 min=32070 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=35280 max_load_reduction_pct=0.515971 moved_to_replicas=129701 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] avg=32768.0000 max=33096 max_rank=1 min=32282 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32873 max_load_reduction_pct=0.498310 moved_to_replicas=97475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] avg=32768.0000 max=33096 max_rank=1 min=32108 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35991 max_load_reduction_pct=0.520952 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[2103, 3, 32765, 129089, 32635, 32762, 32768, 19] avg=32768.0000 max=129089 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.939484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] avg=32768.0000 max=33096 max_rank=1 min=32282 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32873 max_load_reduction_pct=0.498310 moved_to_replicas=97475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[6, 61975, 2, 65526, 32771, 0, 32777, 69087] avg=32768.0000 max=69087 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.108368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[2103, 3, 32765, 129089, 32635, 32762, 32768, 19] avg=32768.0000 max=129089 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.939484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=95993 max_load_reduction_pct=0.743619 moved_to_replicas=95993 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] avg=32768.0000 max=33096 max_rank=1 min=32108 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35991 max_load_reduction_pct=0.520952 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[10, 65535, 32568, 65969, 65259, 0, 12, 32791] avg=32768.0000 max=65969 max_rank=3 min=0 min_rank=5 imbalance_ratio=2.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[2103, 3, 32765, 129089, 32635, 32762, 32768, 19] avg=32768.0000 max=129089 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.939484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[10, 65535, 32568, 65969, 65259, 0, 12, 32791] avg=32768.0000 max=65969 max_rank=3 min=0 min_rank=5 imbalance_ratio=2.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[32758, 65515, 5, 6, 24, 32765, 32774, 98297] avg=32768.0000 max=98297 max_rank=7 min=5 min_rank=2 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[2103, 3, 32765, 129089, 32635, 32762, 32768, 19] avg=32768.0000 max=129089 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.939484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=95993 max_load_reduction_pct=0.743619 moved_to_replicas=95993 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] avg=32768.0000 max=33096 max_rank=1 min=32282 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32873 max_load_reduction_pct=0.498310 moved_to_replicas=97475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[4, 65521, 32771, 5, 32766, 32765, 32760, 65552] avg=32768.0000 max=65552 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=95993 max_load_reduction_pct=0.743619 moved_to_replicas=95993 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] avg=32768.0000 max=33096 max_rank=1 min=32282 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32873 max_load_reduction_pct=0.498310 moved_to_replicas=97475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[10, 65535, 32568, 65969, 65259, 0, 12, 32791] avg=32768.0000 max=65969 max_rank=3 min=0 min_rank=5 imbalance_ratio=2.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[32758, 33096, 32769, 32443, 32443, 32765, 32774, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=97620 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=95993 max_load_reduction_pct=0.743619 moved_to_replicas=95993 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[4, 65521, 32771, 5, 32766, 32765, 32760, 65552] avg=32768.0000 max=65552 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=64881 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] avg=32768.0000 max=33096 max_rank=1 min=32282 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32873 max_load_reduction_pct=0.498310 moved_to_replicas=97475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[2103, 3, 32765, 129089, 32635, 32762, 32768, 19] avg=32768.0000 max=129089 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.939484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[4, 65521, 32771, 5, 32766, 32765, 32760, 65552] avg=32768.0000 max=65552 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[2103, 3, 32765, 129089, 32635, 32762, 32768, 19] avg=32768.0000 max=129089 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.939484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=64881 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[4, 65521, 32771, 5, 32766, 32765, 32760, 65552] avg=32768.0000 max=65552 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[6, 61975, 2, 65526, 32771, 0, 32777, 69087] avg=32768.0000 max=69087 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.108368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=95993 max_load_reduction_pct=0.743619 moved_to_replicas=95993 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[3, 2, 32771, 65530, 3, 32765, 98300, 32770] avg=32768.0000 max=98300 max_rank=6 min=2 min_rank=1 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=64881 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=95993 max_load_reduction_pct=0.743619 moved_to_replicas=95993 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=64881 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[2103, 3, 32765, 129089, 32635, 32762, 32768, 19] avg=32768.0000 max=129089 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.939484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[32436, 33096, 32108, 33096, 32771, 32764, 32777, 33096] avg=32768.0000 max=33096 max_rank=1 min=32108 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=35991 max_load_reduction_pct=0.520952 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[3, 2, 32771, 65530, 3, 32765, 98300, 32770] avg=32768.0000 max=98300 max_rank=6 min=2 min_rank=1 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=95993 max_load_reduction_pct=0.743619 moved_to_replicas=95993 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[4, 65521, 32771, 5, 32766, 32765, 32760, 65552] avg=32768.0000 max=65552 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[3, 2, 32771, 65530, 3, 32765, 98300, 32770] avg=32768.0000 max=98300 max_rank=6 min=2 min_rank=1 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[4, 65521, 32771, 5, 32766, 32765, 32760, 65552] avg=32768.0000 max=65552 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[3, 2, 32771, 65530, 3, 32765, 98300, 32770] avg=32768.0000 max=98300 max_rank=6 min=2 min_rank=1 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[10, 65535, 32568, 65969, 65259, 0, 12, 32791] avg=32768.0000 max=65969 max_rank=3 min=0 min_rank=5 imbalance_ratio=2.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=25 tokens_per_ep_rank=[1, 32771, 98279, 32765, 65583, 3, 32735, 7] avg=32768.0000 max=98279 max_rank=2 min=1 min_rank=0 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=64881 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[4, 65521, 32771, 5, 32766, 32765, 32760, 65552] avg=32768.0000 max=65552 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=64881 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[32282, 33096, 32568, 33096, 33096, 32764, 32451, 32791] avg=32768.0000 max=33096 max_rank=1 min=32282 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32873 max_load_reduction_pct=0.498310 moved_to_replicas=97475 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=25 tokens_per_ep_rank=[1, 32771, 98279, 32765, 65583, 3, 32735, 7] avg=32768.0000 max=98279 max_rank=2 min=1 min_rank=0 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=97670 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=64881 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[3, 2, 32771, 65530, 3, 32765, 98300, 32770] avg=32768.0000 max=98300 max_rank=6 min=2 min_rank=1 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=97670 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=25 tokens_per_ep_rank=[1, 32771, 98279, 32765, 65583, 3, 32735, 7] avg=32768.0000 max=98279 max_rank=2 min=1 min_rank=0 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=25 tokens_per_ep_rank=[1, 32771, 98279, 32765, 65583, 3, 32735, 7] avg=32768.0000 max=98279 max_rank=2 min=1 min_rank=0 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[3, 2, 32771, 65530, 3, 32765, 98300, 32770] avg=32768.0000 max=98300 max_rank=6 min=2 min_rank=1 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[2103, 3, 32765, 129089, 32635, 32762, 32768, 19] avg=32768.0000 max=129089 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.939484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=26 tokens_per_ep_rank=[19, 32768, 32769, 32809, 32763, 98252, 32764, 0] avg=32768.0000 max=98252 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=97670 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=97670 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[3, 2, 32771, 65530, 3, 32765, 98300, 32770] avg=32768.0000 max=98300 max_rank=6 min=2 min_rank=1 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=26 tokens_per_ep_rank=[19, 32768, 32769, 32809, 32763, 98252, 32764, 0] avg=32768.0000 max=98252 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[32568, 32767, 32765, 33096, 32635, 32762, 32768, 32783] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=95993 max_load_reduction_pct=0.743619 moved_to_replicas=95993 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=26 tokens_per_ep_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=65156 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=25 tokens_per_ep_rank=[1, 32771, 98279, 32765, 65583, 3, 32735, 7] avg=32768.0000 max=98279 max_rank=2 min=1 min_rank=0 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=26 tokens_per_ep_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=65156 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=26 tokens_per_ep_rank=[19, 32768, 32769, 32809, 32763, 98252, 32764, 0] avg=32768.0000 max=98252 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=26 tokens_per_ep_rank=[19, 32768, 32769, 32809, 32763, 98252, 32764, 0] avg=32768.0000 max=98252 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=25 tokens_per_ep_rank=[1, 32771, 98279, 32765, 65583, 3, 32735, 7] avg=32768.0000 max=98279 max_rank=2 min=1 min_rank=0 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[4, 65521, 32771, 5, 32766, 32765, 32760, 65552] avg=32768.0000 max=65552 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=27 tokens_per_ep_rank=[32770, 41, 65532, 32727, 32769, 32771, 65530, 4] avg=32768.0000 max=65532 max_rank=2 min=4 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=97670 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=26 tokens_per_ep_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=65156 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=26 tokens_per_ep_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=65156 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=25 tokens_per_ep_rank=[1, 32771, 98279, 32765, 65583, 3, 32735, 7] avg=32768.0000 max=98279 max_rank=2 min=1 min_rank=0 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=97670 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=27 tokens_per_ep_rank=[32770, 41, 65532, 32727, 32769, 32771, 65530, 4] avg=32768.0000 max=65532 max_rank=2 min=4 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=27 tokens_per_ep_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[32460, 33096, 32771, 32430, 32766, 32765, 32760, 33096] avg=32768.0000 max=33096 max_rank=1 min=32430 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=64881 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=97670 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=26 tokens_per_ep_rank=[19, 32768, 32769, 32809, 32763, 98252, 32764, 0] avg=32768.0000 max=98252 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=27 tokens_per_ep_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=27 tokens_per_ep_rank=[32770, 41, 65532, 32727, 32769, 32771, 65530, 4] avg=32768.0000 max=65532 max_rank=2 min=4 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=26 tokens_per_ep_rank=[19, 32768, 32769, 32809, 32763, 98252, 32764, 0] avg=32768.0000 max=98252 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=27 tokens_per_ep_rank=[32770, 41, 65532, 32727, 32769, 32771, 65530, 4] avg=32768.0000 max=65532 max_rank=2 min=4 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 65526, 32762, 32778, 6, 0, 65536] avg=32768.0000 max=65536 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=26 tokens_per_ep_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=65156 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[3, 2, 32771, 65530, 3, 32765, 98300, 32770] avg=32768.0000 max=98300 max_rank=6 min=2 min_rank=1 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=26 tokens_per_ep_rank=[19, 32768, 32769, 32809, 32763, 98252, 32764, 0] avg=32768.0000 max=98252 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=27 tokens_per_ep_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=27 tokens_per_ep_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=26 tokens_per_ep_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=65156 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 65526, 32762, 32778, 6, 0, 65536] avg=32768.0000 max=65536 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] avg=32768.0000 max=33096 max_rank=2 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[32443, 32766, 32771, 33096, 32437, 32765, 33096, 32770] avg=32768.0000 max=33096 max_rank=3 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65204 max_load_reduction_pct=0.663316 moved_to_replicas=97638 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=26 tokens_per_ep_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=65156 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=27 tokens_per_ep_rank=[32770, 41, 65532, 32727, 32769, 32771, 65530, 4] avg=32768.0000 max=65532 max_rank=2 min=4 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] avg=32768.0000 max=33096 max_rank=2 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 65526, 32762, 32778, 6, 0, 65536] avg=32768.0000 max=65536 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 65526, 32762, 32778, 6, 0, 65536] avg=32768.0000 max=65536 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=27 tokens_per_ep_rank=[32770, 41, 65532, 32727, 32769, 32771, 65530, 4] avg=32768.0000 max=65532 max_rank=2 min=4 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=29 tokens_per_ep_rank=[6, 65515, 132, 37831, 65533, 32760, 27600, 32767] avg=32768.0000 max=65533 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=27 tokens_per_ep_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] avg=32768.0000 max=33096 max_rank=2 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=27 tokens_per_ep_rank=[32770, 41, 65532, 32727, 32769, 32771, 65530, 4] avg=32768.0000 max=65532 max_rank=2 min=4 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=25 tokens_per_ep_rank=[1, 32771, 98279, 32765, 65583, 3, 32735, 7] avg=32768.0000 max=98279 max_rank=2 min=1 min_rank=0 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=27 tokens_per_ep_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] avg=32768.0000 max=33096 max_rank=2 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=29 tokens_per_ep_rank=[6, 65515, 132, 37831, 65533, 32760, 27600, 32767] avg=32768.0000 max=65533 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=29 tokens_per_ep_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] avg=32768.0000 max=33096 max_rank=1 min=32335 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=69591 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=27 tokens_per_ep_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=25 tokens_per_ep_rank=[32765, 32771, 33096, 32765, 33096, 32422, 32735, 32494] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=97670 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=29 tokens_per_ep_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] avg=32768.0000 max=33096 max_rank=1 min=32335 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=69591 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 65526, 32762, 32778, 6, 0, 65536] avg=32768.0000 max=65536 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=29 tokens_per_ep_rank=[6, 65515, 132, 37831, 65533, 32760, 27600, 32767] avg=32768.0000 max=65533 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 65526, 32762, 32778, 6, 0, 65536] avg=32768.0000 max=65536 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=29 tokens_per_ep_rank=[6, 65515, 132, 37831, 65533, 32760, 27600, 32767] avg=32768.0000 max=65533 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=30 tokens_per_ep_rank=[31, 32768, 32768, 32871, 98292, 3, 32767, 32644] avg=32768.0000 max=98292 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] avg=32768.0000 max=33096 max_rank=2 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=29 tokens_per_ep_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] avg=32768.0000 max=33096 max_rank=1 min=32335 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=69591 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 65526, 32762, 32778, 6, 0, 65536] avg=32768.0000 max=65536 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=26 tokens_per_ep_rank=[19, 32768, 32769, 32809, 32763, 98252, 32764, 0] avg=32768.0000 max=98252 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=30 tokens_per_ep_rank=[31, 32768, 32768, 32871, 98292, 3, 32767, 32644] avg=32768.0000 max=98292 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] avg=32768.0000 max=33096 max_rank=2 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=29 tokens_per_ep_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] avg=32768.0000 max=33096 max_rank=1 min=32335 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=69591 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=30 tokens_per_ep_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] avg=32768.0000 max=33096 max_rank=4 min=32463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=65196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] avg=32768.0000 max=33096 max_rank=2 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=26 tokens_per_ep_rank=[32411, 32768, 32769, 32809, 32763, 33096, 32764, 32764] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=65156 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=30 tokens_per_ep_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] avg=32768.0000 max=33096 max_rank=4 min=32463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=65196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=29 tokens_per_ep_rank=[6, 65515, 132, 37831, 65533, 32760, 27600, 32767] avg=32768.0000 max=65533 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=30 tokens_per_ep_rank=[31, 32768, 32768, 32871, 98292, 3, 32767, 32644] avg=32768.0000 max=98292 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=29 tokens_per_ep_rank=[6, 65515, 132, 37831, 65533, 32760, 27600, 32767] avg=32768.0000 max=65533 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=30 tokens_per_ep_rank=[31, 32768, 32768, 32871, 98292, 3, 32767, 32644] avg=32768.0000 max=98292 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=31 tokens_per_ep_rank=[9, 34, 32757, 32751, 65527, 32765, 3, 98298] avg=32768.0000 max=98298 max_rank=7 min=3 min_rank=6 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=30 tokens_per_ep_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] avg=32768.0000 max=33096 max_rank=4 min=32463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=65196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=29 tokens_per_ep_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] avg=32768.0000 max=33096 max_rank=1 min=32335 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=69591 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=29 tokens_per_ep_rank=[6, 65515, 132, 37831, 65533, 32760, 27600, 32767] avg=32768.0000 max=65533 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=29 tokens_per_ep_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] avg=32768.0000 max=33096 max_rank=1 min=32335 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=69591 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=27 tokens_per_ep_rank=[32770, 41, 65532, 32727, 32769, 32771, 65530, 4] avg=32768.0000 max=65532 max_rank=2 min=4 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=31 tokens_per_ep_rank=[9, 34, 32757, 32751, 65527, 32765, 3, 98298] avg=32768.0000 max=98298 max_rank=7 min=3 min_rank=6 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=30 tokens_per_ep_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] avg=32768.0000 max=33096 max_rank=4 min=32463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=65196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=31 tokens_per_ep_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] avg=32768.0000 max=33096 max_rank=4 min=32447 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=29 tokens_per_ep_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] avg=32768.0000 max=33096 max_rank=1 min=32335 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=69591 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=27 tokens_per_ep_rank=[32770, 32475, 33096, 32727, 32769, 32771, 33096, 32440] avg=32768.0000 max=33096 max_rank=2 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=31 tokens_per_ep_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] avg=32768.0000 max=33096 max_rank=4 min=32447 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=31 tokens_per_ep_rank=[9, 34, 32757, 32751, 65527, 32765, 3, 98298] avg=32768.0000 max=98298 max_rank=7 min=3 min_rank=6 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=30 tokens_per_ep_rank=[31, 32768, 32768, 32871, 98292, 3, 32767, 32644] avg=32768.0000 max=98292 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=30 tokens_per_ep_rank=[31, 32768, 32768, 32871, 98292, 3, 32767, 32644] avg=32768.0000 max=98292 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=32 tokens_per_ep_rank=[32773, 9, 65542, 1, 65518, 65535, 32766, 0] avg=32768.0000 max=65542 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=31 tokens_per_ep_rank=[9, 34, 32757, 32751, 65527, 32765, 3, 98298] avg=32768.0000 max=98298 max_rank=7 min=3 min_rank=6 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=31 tokens_per_ep_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] avg=32768.0000 max=33096 max_rank=4 min=32447 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=30 tokens_per_ep_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] avg=32768.0000 max=33096 max_rank=4 min=32463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=65196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=30 tokens_per_ep_rank=[31, 32768, 32768, 32871, 98292, 3, 32767, 32644] avg=32768.0000 max=98292 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=30 tokens_per_ep_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] avg=32768.0000 max=33096 max_rank=4 min=32463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=65196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=32 tokens_per_ep_rank=[32773, 9, 65542, 1, 65518, 65535, 32766, 0] avg=32768.0000 max=65542 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 65526, 32762, 32778, 6, 0, 65536] avg=32768.0000 max=65536 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=32 tokens_per_ep_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] avg=32768.0000 max=33096 max_rank=2 min=32431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32446 max_load_reduction_pct=0.495041 moved_to_replicas=97307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=31 tokens_per_ep_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] avg=32768.0000 max=33096 max_rank=4 min=32447 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=30 tokens_per_ep_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] avg=32768.0000 max=33096 max_rank=4 min=32463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=65196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=32 tokens_per_ep_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] avg=32768.0000 max=33096 max_rank=2 min=32431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32446 max_load_reduction_pct=0.495041 moved_to_replicas=97307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=32 tokens_per_ep_rank=[32773, 9, 65542, 1, 65518, 65535, 32766, 0] avg=32768.0000 max=65542 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=28 tokens_per_ep_rank=[32768, 32768, 33096, 32762, 32778, 32436, 32440, 33096] avg=32768.0000 max=33096 max_rank=2 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64870 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=31 tokens_per_ep_rank=[9, 34, 32757, 32751, 65527, 32765, 3, 98298] avg=32768.0000 max=98298 max_rank=7 min=3 min_rank=6 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=31 tokens_per_ep_rank=[9, 34, 32757, 32751, 65527, 32765, 3, 98298] avg=32768.0000 max=98298 max_rank=7 min=3 min_rank=6 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=33 tokens_per_ep_rank=[65532, 5, 32765, 8, 32748, 32765, 98320, 1] avg=32768.0000 max=98320 max_rank=6 min=1 min_rank=7 imbalance_ratio=3.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=32 tokens_per_ep_rank=[32773, 9, 65542, 1, 65518, 65535, 32766, 0] avg=32768.0000 max=65542 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=32 tokens_per_ep_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] avg=32768.0000 max=33096 max_rank=2 min=32431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32446 max_load_reduction_pct=0.495041 moved_to_replicas=97307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=31 tokens_per_ep_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] avg=32768.0000 max=33096 max_rank=4 min=32447 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=31 tokens_per_ep_rank=[9, 34, 32757, 32751, 65527, 32765, 3, 98298] avg=32768.0000 max=98298 max_rank=7 min=3 min_rank=6 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=31 tokens_per_ep_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] avg=32768.0000 max=33096 max_rank=4 min=32447 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=33 tokens_per_ep_rank=[65532, 5, 32765, 8, 32748, 32765, 98320, 1] avg=32768.0000 max=98320 max_rank=6 min=1 min_rank=7 imbalance_ratio=3.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65224 max_load_reduction_pct=0.663385 moved_to_replicas=97660 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=29 tokens_per_ep_rank=[6, 65515, 132, 37831, 65533, 32760, 27600, 32767] avg=32768.0000 max=65533 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=32 tokens_per_ep_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] avg=32768.0000 max=33096 max_rank=2 min=32431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32446 max_load_reduction_pct=0.495041 moved_to_replicas=97307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65224 max_load_reduction_pct=0.663385 moved_to_replicas=97660 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=31 tokens_per_ep_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] avg=32768.0000 max=33096 max_rank=4 min=32447 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=33 tokens_per_ep_rank=[65532, 5, 32765, 8, 32748, 32765, 98320, 1] avg=32768.0000 max=98320 max_rank=6 min=1 min_rank=7 imbalance_ratio=3.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=29 tokens_per_ep_rank=[32443, 33096, 32551, 33096, 33096, 32760, 32335, 32767] avg=32768.0000 max=33096 max_rank=1 min=32335 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=69591 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=32 tokens_per_ep_rank=[32773, 9, 65542, 1, 65518, 65535, 32766, 0] avg=32768.0000 max=65542 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=32 tokens_per_ep_rank=[32773, 9, 65542, 1, 65518, 65535, 32766, 0] avg=32768.0000 max=65542 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=34 tokens_per_ep_rank=[65484, 65536, 32391, 3, 9, 0, 1, 98720] avg=32768.0000 max=98720 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.012695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=33 tokens_per_ep_rank=[65532, 5, 32765, 8, 32748, 32765, 98320, 1] avg=32768.0000 max=98320 max_rank=6 min=1 min_rank=7 imbalance_ratio=3.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65224 max_load_reduction_pct=0.663385 moved_to_replicas=97660 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=32 tokens_per_ep_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] avg=32768.0000 max=33096 max_rank=2 min=32431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32446 max_load_reduction_pct=0.495041 moved_to_replicas=97307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=32 tokens_per_ep_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] avg=32768.0000 max=33096 max_rank=2 min=32431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32446 max_load_reduction_pct=0.495041 moved_to_replicas=97307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=34 tokens_per_ep_rank=[65484, 65536, 32391, 3, 9, 0, 1, 98720] avg=32768.0000 max=98720 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.012695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=34 tokens_per_ep_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65624 max_load_reduction_pct=0.664749 moved_to_replicas=130452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=32 tokens_per_ep_rank=[32773, 9, 65542, 1, 65518, 65535, 32766, 0] avg=32768.0000 max=65542 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=30 tokens_per_ep_rank=[31, 32768, 32768, 32871, 98292, 3, 32767, 32644] avg=32768.0000 max=98292 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65224 max_load_reduction_pct=0.663385 moved_to_replicas=97660 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=34 tokens_per_ep_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65624 max_load_reduction_pct=0.664749 moved_to_replicas=130452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=34 tokens_per_ep_rank=[65484, 65536, 32391, 3, 9, 0, 1, 98720] avg=32768.0000 max=98720 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.012695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=32 tokens_per_ep_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] avg=32768.0000 max=33096 max_rank=2 min=32431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32446 max_load_reduction_pct=0.495041 moved_to_replicas=97307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=30 tokens_per_ep_rank=[32463, 32768, 32768, 32871, 33096, 32767, 32767, 32644] avg=32768.0000 max=33096 max_rank=4 min=32463 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=65196 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=33 tokens_per_ep_rank=[65532, 5, 32765, 8, 32748, 32765, 98320, 1] avg=32768.0000 max=98320 max_rank=6 min=1 min_rank=7 imbalance_ratio=3.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=33 tokens_per_ep_rank=[65532, 5, 32765, 8, 32748, 32765, 98320, 1] avg=32768.0000 max=98320 max_rank=6 min=1 min_rank=7 imbalance_ratio=3.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=35 tokens_per_ep_rank=[6, 3, 32774, 131067, 32761, 0, 65533, 0] avg=32768.0000 max=131067 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=34 tokens_per_ep_rank=[65484, 65536, 32391, 3, 9, 0, 1, 98720] avg=32768.0000 max=98720 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.012695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=34 tokens_per_ep_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65624 max_load_reduction_pct=0.664749 moved_to_replicas=130452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65224 max_load_reduction_pct=0.663385 moved_to_replicas=97660 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65224 max_load_reduction_pct=0.663385 moved_to_replicas=97660 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=35 tokens_per_ep_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] avg=32768.0000 max=33096 max_rank=3 min=32443 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97971 max_load_reduction_pct=0.747488 moved_to_replicas=130408 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=35 tokens_per_ep_rank=[6, 3, 32774, 131067, 32761, 0, 65533, 0] avg=32768.0000 max=131067 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=33 tokens_per_ep_rank=[65532, 5, 32765, 8, 32748, 32765, 98320, 1] avg=32768.0000 max=98320 max_rank=6 min=1 min_rank=7 imbalance_ratio=3.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=31 tokens_per_ep_rank=[9, 34, 32757, 32751, 65527, 32765, 3, 98298] avg=32768.0000 max=98298 max_rank=7 min=3 min_rank=6 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=34 tokens_per_ep_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65624 max_load_reduction_pct=0.664749 moved_to_replicas=130452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=35 tokens_per_ep_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] avg=32768.0000 max=33096 max_rank=3 min=32443 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97971 max_load_reduction_pct=0.747488 moved_to_replicas=130408 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=35 tokens_per_ep_rank=[6, 3, 32774, 131067, 32761, 0, 65533, 0] avg=32768.0000 max=131067 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65224 max_load_reduction_pct=0.663385 moved_to_replicas=97660 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=31 tokens_per_ep_rank=[32447, 32465, 32757, 32751, 33096, 32765, 32767, 33096] avg=32768.0000 max=33096 max_rank=4 min=32447 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=34 tokens_per_ep_rank=[65484, 65536, 32391, 3, 9, 0, 1, 98720] avg=32768.0000 max=98720 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.012695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=34 tokens_per_ep_rank=[65484, 65536, 32391, 3, 9, 0, 1, 98720] avg=32768.0000 max=98720 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.012695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=36 tokens_per_ep_rank=[32768, 65535, 32768, 32765, 3, 32774, 3, 65528] avg=32768.0000 max=65535 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=35 tokens_per_ep_rank=[6, 3, 32774, 131067, 32761, 0, 65533, 0] avg=32768.0000 max=131067 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=35 tokens_per_ep_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] avg=32768.0000 max=33096 max_rank=3 min=32443 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97971 max_load_reduction_pct=0.747488 moved_to_replicas=130408 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=34 tokens_per_ep_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65624 max_load_reduction_pct=0.664749 moved_to_replicas=130452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=36 tokens_per_ep_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64871 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=34 tokens_per_ep_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65624 max_load_reduction_pct=0.664749 moved_to_replicas=130452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=36 tokens_per_ep_rank=[32768, 65535, 32768, 32765, 3, 32774, 3, 65528] avg=32768.0000 max=65535 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=34 tokens_per_ep_rank=[65484, 65536, 32391, 3, 9, 0, 1, 98720] avg=32768.0000 max=98720 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.012695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=32 tokens_per_ep_rank=[32773, 9, 65542, 1, 65518, 65535, 32766, 0] avg=32768.0000 max=65542 max_rank=2 min=0 min_rank=7 imbalance_ratio=2.000183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=35 tokens_per_ep_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] avg=32768.0000 max=33096 max_rank=3 min=32443 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97971 max_load_reduction_pct=0.747488 moved_to_replicas=130408 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=36 tokens_per_ep_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64871 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=36 tokens_per_ep_rank=[32768, 65535, 32768, 32765, 3, 32774, 3, 65528] avg=32768.0000 max=65535 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=34 tokens_per_ep_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65624 max_load_reduction_pct=0.664749 moved_to_replicas=130452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=32 tokens_per_ep_rank=[32773, 32431, 33096, 32440, 33096, 33096, 32766, 32446] avg=32768.0000 max=33096 max_rank=2 min=32431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32446 max_load_reduction_pct=0.495041 moved_to_replicas=97307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=35 tokens_per_ep_rank=[6, 3, 32774, 131067, 32761, 0, 65533, 0] avg=32768.0000 max=131067 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=37 tokens_per_ep_rank=[32765, 34117, 9, 107, 6, 0, 96842, 98298] avg=32768.0000 max=98298 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=35 tokens_per_ep_rank=[6, 3, 32774, 131067, 32761, 0, 65533, 0] avg=32768.0000 max=131067 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=36 tokens_per_ep_rank=[32768, 65535, 32768, 32765, 3, 32774, 3, 65528] avg=32768.0000 max=65535 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=36 tokens_per_ep_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64871 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=37 tokens_per_ep_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32110 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=129969 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=35 tokens_per_ep_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] avg=32768.0000 max=33096 max_rank=3 min=32443 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97971 max_load_reduction_pct=0.747488 moved_to_replicas=130408 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=37 tokens_per_ep_rank=[32765, 34117, 9, 107, 6, 0, 96842, 98298] avg=32768.0000 max=98298 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=35 tokens_per_ep_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] avg=32768.0000 max=33096 max_rank=3 min=32443 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97971 max_load_reduction_pct=0.747488 moved_to_replicas=130408 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=35 tokens_per_ep_rank=[6, 3, 32774, 131067, 32761, 0, 65533, 0] avg=32768.0000 max=131067 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=33 tokens_per_ep_rank=[65532, 5, 32765, 8, 32748, 32765, 98320, 1] avg=32768.0000 max=98320 max_rank=6 min=1 min_rank=7 imbalance_ratio=3.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=36 tokens_per_ep_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64871 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=37 tokens_per_ep_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32110 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=129969 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=37 tokens_per_ep_rank=[32765, 34117, 9, 107, 6, 0, 96842, 98298] avg=32768.0000 max=98298 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=35 tokens_per_ep_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] avg=32768.0000 max=33096 max_rank=3 min=32443 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97971 max_load_reduction_pct=0.747488 moved_to_replicas=130408 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=33 tokens_per_ep_rank=[33096, 32465, 32765, 32444, 32748, 32765, 33096, 32765] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65224 max_load_reduction_pct=0.663385 moved_to_replicas=97660 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=38 tokens_per_ep_rank=[32768, 0, 65995, 3, 65074, 32765, 6, 65533] avg=32768.0000 max=65995 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.014008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=36 tokens_per_ep_rank=[32768, 65535, 32768, 32765, 3, 32774, 3, 65528] avg=32768.0000 max=65535 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=36 tokens_per_ep_rank=[32768, 65535, 32768, 32765, 3, 32774, 3, 65528] avg=32768.0000 max=65535 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=37 tokens_per_ep_rank=[32765, 34117, 9, 107, 6, 0, 96842, 98298] avg=32768.0000 max=98298 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=37 tokens_per_ep_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32110 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=129969 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=38 tokens_per_ep_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] avg=32768.0000 max=33096 max_rank=2 min=32116 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32899 max_load_reduction_pct=0.498507 moved_to_replicas=97314 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=36 tokens_per_ep_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64871 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=38 tokens_per_ep_rank=[32768, 0, 65995, 3, 65074, 32765, 6, 65533] avg=32768.0000 max=65995 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.014008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=36 tokens_per_ep_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64871 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=36 tokens_per_ep_rank=[32768, 65535, 32768, 32765, 3, 32774, 3, 65528] avg=32768.0000 max=65535 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=34 tokens_per_ep_rank=[65484, 65536, 32391, 3, 9, 0, 1, 98720] avg=32768.0000 max=98720 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.012695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=37 tokens_per_ep_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32110 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=129969 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=38 tokens_per_ep_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] avg=32768.0000 max=33096 max_rank=2 min=32116 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32899 max_load_reduction_pct=0.498507 moved_to_replicas=97314 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=38 tokens_per_ep_rank=[32768, 0, 65995, 3, 65074, 32765, 6, 65533] avg=32768.0000 max=65995 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.014008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=36 tokens_per_ep_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64871 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 3, 65534, 32771, 32765] avg=32768.0000 max=65534 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=34 tokens_per_ep_rank=[33096, 33096, 32391, 32487, 32449, 32764, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65624 max_load_reduction_pct=0.664749 moved_to_replicas=130452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=37 tokens_per_ep_rank=[32765, 34117, 9, 107, 6, 0, 96842, 98298] avg=32768.0000 max=98298 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=37 tokens_per_ep_rank=[32765, 34117, 9, 107, 6, 0, 96842, 98298] avg=32768.0000 max=98298 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=38 tokens_per_ep_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] avg=32768.0000 max=33096 max_rank=2 min=32116 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32899 max_load_reduction_pct=0.498507 moved_to_replicas=97314 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=38 tokens_per_ep_rank=[32768, 0, 65995, 3, 65074, 32765, 6, 65533] avg=32768.0000 max=65995 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.014008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32438 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 3, 65534, 32771, 32765] avg=32768.0000 max=65534 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=37 tokens_per_ep_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32110 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=129969 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=37 tokens_per_ep_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32110 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=129969 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=37 tokens_per_ep_rank=[32765, 34117, 9, 107, 6, 0, 96842, 98298] avg=32768.0000 max=98298 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=35 tokens_per_ep_rank=[6, 3, 32774, 131067, 32761, 0, 65533, 0] avg=32768.0000 max=131067 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=38 tokens_per_ep_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] avg=32768.0000 max=33096 max_rank=2 min=32116 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32899 max_load_reduction_pct=0.498507 moved_to_replicas=97314 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32438 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 3, 65534, 32771, 32765] avg=32768.0000 max=65534 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=40 tokens_per_ep_rank=[102, 65533, 32764, 32768, 6, 65531, 65437, 3] avg=32768.0000 max=65533 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=37 tokens_per_ep_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32110 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=129969 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=35 tokens_per_ep_rank=[32443, 32446, 32774, 33096, 32761, 32764, 33096, 32764] avg=32768.0000 max=33096 max_rank=3 min=32443 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97971 max_load_reduction_pct=0.747488 moved_to_replicas=130408 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=38 tokens_per_ep_rank=[32768, 0, 65995, 3, 65074, 32765, 6, 65533] avg=32768.0000 max=65995 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.014008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=38 tokens_per_ep_rank=[32768, 0, 65995, 3, 65074, 32765, 6, 65533] avg=32768.0000 max=65995 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.014008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32438 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 3, 65534, 32771, 32765] avg=32768.0000 max=65534 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=40 tokens_per_ep_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97213 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=40 tokens_per_ep_rank=[102, 65533, 32764, 32768, 6, 65531, 65437, 3] avg=32768.0000 max=65533 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=38 tokens_per_ep_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] avg=32768.0000 max=33096 max_rank=2 min=32116 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32899 max_load_reduction_pct=0.498507 moved_to_replicas=97314 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=38 tokens_per_ep_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] avg=32768.0000 max=33096 max_rank=2 min=32116 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32899 max_load_reduction_pct=0.498507 moved_to_replicas=97314 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=38 tokens_per_ep_rank=[32768, 0, 65995, 3, 65074, 32765, 6, 65533] avg=32768.0000 max=65995 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.014008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=36 tokens_per_ep_rank=[32768, 65535, 32768, 32765, 3, 32774, 3, 65528] avg=32768.0000 max=65535 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32438 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=40 tokens_per_ep_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97213 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=40 tokens_per_ep_rank=[102, 65533, 32764, 32768, 6, 65531, 65437, 3] avg=32768.0000 max=65533 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=38 tokens_per_ep_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] avg=32768.0000 max=33096 max_rank=2 min=32116 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32899 max_load_reduction_pct=0.498507 moved_to_replicas=97314 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=41 tokens_per_ep_rank=[65531, 32762, 32770, 3, 3, 98294, 11, 32770] avg=32768.0000 max=98294 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=36 tokens_per_ep_rank=[32768, 33096, 32768, 32765, 32442, 32774, 32435, 33096] avg=32768.0000 max=33096 max_rank=1 min=32435 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64871 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 3, 65534, 32771, 32765] avg=32768.0000 max=65534 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 3, 65534, 32771, 32765] avg=32768.0000 max=65534 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=40 tokens_per_ep_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97213 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=41 tokens_per_ep_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=41 tokens_per_ep_rank=[65531, 32762, 32770, 3, 3, 98294, 11, 32770] avg=32768.0000 max=98294 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=40 tokens_per_ep_rank=[102, 65533, 32764, 32768, 6, 65531, 65437, 3] avg=32768.0000 max=65533 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32438 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32438 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 3, 65534, 32771, 32765] avg=32768.0000 max=65534 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=37 tokens_per_ep_rank=[32765, 34117, 9, 107, 6, 0, 96842, 98298] avg=32768.0000 max=98298 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=41 tokens_per_ep_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=40 tokens_per_ep_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97213 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=41 tokens_per_ep_rank=[65531, 32762, 32770, 3, 3, 98294, 11, 32770] avg=32768.0000 max=98294 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=42 tokens_per_ep_rank=[378, 32428, 3, 65501, 65537, 32766, 32766, 32765] avg=32768.0000 max=65537 max_rank=4 min=3 min_rank=2 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32438 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=40 tokens_per_ep_rank=[102, 65533, 32764, 32768, 6, 65531, 65437, 3] avg=32768.0000 max=65533 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=37 tokens_per_ep_rank=[32765, 33096, 32773, 32110, 32444, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32110 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65202 max_load_reduction_pct=0.663310 moved_to_replicas=129969 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=40 tokens_per_ep_rank=[102, 65533, 32764, 32768, 6, 65531, 65437, 3] avg=32768.0000 max=65533 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=41 tokens_per_ep_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=42 tokens_per_ep_rank=[378, 32428, 3, 65501, 65537, 32766, 32766, 32765] avg=32768.0000 max=65537 max_rank=4 min=3 min_rank=2 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=42 tokens_per_ep_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] avg=32768.0000 max=33096 max_rank=3 min=32428 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=64846 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=40 tokens_per_ep_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97213 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=41 tokens_per_ep_rank=[65531, 32762, 32770, 3, 3, 98294, 11, 32770] avg=32768.0000 max=98294 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=40 tokens_per_ep_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97213 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=40 tokens_per_ep_rank=[102, 65533, 32764, 32768, 6, 65531, 65437, 3] avg=32768.0000 max=65533 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=38 tokens_per_ep_rank=[32768, 0, 65995, 3, 65074, 32765, 6, 65533] avg=32768.0000 max=65995 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.014008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=42 tokens_per_ep_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] avg=32768.0000 max=33096 max_rank=3 min=32428 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=64846 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=42 tokens_per_ep_rank=[378, 32428, 3, 65501, 65537, 32766, 32766, 32765] avg=32768.0000 max=65537 max_rank=4 min=3 min_rank=2 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=41 tokens_per_ep_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=43 tokens_per_ep_rank=[46, 32766, 5, 19620, 65534, 95668, 15600, 32905] avg=32768.0000 max=95668 max_rank=5 min=5 min_rank=2 imbalance_ratio=2.919556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=41 tokens_per_ep_rank=[65531, 32762, 32770, 3, 3, 98294, 11, 32770] avg=32768.0000 max=98294 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=40 tokens_per_ep_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97213 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=38 tokens_per_ep_rank=[32768, 32764, 33096, 32116, 33096, 32765, 32443, 33096] avg=32768.0000 max=33096 max_rank=2 min=32116 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32899 max_load_reduction_pct=0.498507 moved_to_replicas=97314 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=41 tokens_per_ep_rank=[65531, 32762, 32770, 3, 3, 98294, 11, 32770] avg=32768.0000 max=98294 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=42 tokens_per_ep_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] avg=32768.0000 max=33096 max_rank=3 min=32428 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=64846 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=43 tokens_per_ep_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] avg=32768.0000 max=33097 max_rank=3 min=31320 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62571 max_load_reduction_pct=0.654043 moved_to_replicas=95010 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=43 tokens_per_ep_rank=[46, 32766, 5, 19620, 65534, 95668, 15600, 32905] avg=32768.0000 max=95668 max_rank=5 min=5 min_rank=2 imbalance_ratio=2.919556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=41 tokens_per_ep_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=42 tokens_per_ep_rank=[378, 32428, 3, 65501, 65537, 32766, 32766, 32765] avg=32768.0000 max=65537 max_rank=4 min=3 min_rank=2 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=41 tokens_per_ep_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=41 tokens_per_ep_rank=[65531, 32762, 32770, 3, 3, 98294, 11, 32770] avg=32768.0000 max=98294 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 3, 65534, 32771, 32765] avg=32768.0000 max=65534 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=43 tokens_per_ep_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] avg=32768.0000 max=33097 max_rank=3 min=31320 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62571 max_load_reduction_pct=0.654043 moved_to_replicas=95010 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=43 tokens_per_ep_rank=[46, 32766, 5, 19620, 65534, 95668, 15600, 32905] avg=32768.0000 max=95668 max_rank=5 min=5 min_rank=2 imbalance_ratio=2.919556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=44 tokens_per_ep_rank=[65490, 9, 32767, 3, 32766, 65530, 32770, 32809] avg=32768.0000 max=65530 max_rank=5 min=3 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=42 tokens_per_ep_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] avg=32768.0000 max=33096 max_rank=3 min=32428 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=64846 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=41 tokens_per_ep_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=39 tokens_per_ep_rank=[32768, 32764, 32768, 32771, 32441, 33096, 32771, 32765] avg=32768.0000 max=33096 max_rank=5 min=32441 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=32438 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=42 tokens_per_ep_rank=[378, 32428, 3, 65501, 65537, 32766, 32766, 32765] avg=32768.0000 max=65537 max_rank=4 min=3 min_rank=2 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=42 tokens_per_ep_rank=[378, 32428, 3, 65501, 65537, 32766, 32766, 32765] avg=32768.0000 max=65537 max_rank=4 min=3 min_rank=2 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=43 tokens_per_ep_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] avg=32768.0000 max=33097 max_rank=3 min=31320 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62571 max_load_reduction_pct=0.654043 moved_to_replicas=95010 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=44 tokens_per_ep_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64828 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=44 tokens_per_ep_rank=[65490, 9, 32767, 3, 32766, 65530, 32770, 32809] avg=32768.0000 max=65530 max_rank=5 min=3 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=42 tokens_per_ep_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] avg=32768.0000 max=33096 max_rank=3 min=32428 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=64846 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=43 tokens_per_ep_rank=[46, 32766, 5, 19620, 65534, 95668, 15600, 32905] avg=32768.0000 max=95668 max_rank=5 min=5 min_rank=2 imbalance_ratio=2.919556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=42 tokens_per_ep_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] avg=32768.0000 max=33096 max_rank=3 min=32428 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=64846 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=42 tokens_per_ep_rank=[378, 32428, 3, 65501, 65537, 32766, 32766, 32765] avg=32768.0000 max=65537 max_rank=4 min=3 min_rank=2 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=40 tokens_per_ep_rank=[102, 65533, 32764, 32768, 6, 65531, 65437, 3] avg=32768.0000 max=65533 max_rank=1 min=3 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=44 tokens_per_ep_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64828 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=44 tokens_per_ep_rank=[65490, 9, 32767, 3, 32766, 65530, 32770, 32809] avg=32768.0000 max=65530 max_rank=5 min=3 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=45 tokens_per_ep_rank=[6, 51036, 32884, 822, 79094, 32765, 32766, 32771] avg=32768.0000 max=79094 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.413757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=43 tokens_per_ep_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] avg=32768.0000 max=33097 max_rank=3 min=31320 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62571 max_load_reduction_pct=0.654043 moved_to_replicas=95010 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=42 tokens_per_ep_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] avg=32768.0000 max=33096 max_rank=3 min=32428 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=64846 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=43 tokens_per_ep_rank=[46, 32766, 5, 19620, 65534, 95668, 15600, 32905] avg=32768.0000 max=95668 max_rank=5 min=5 min_rank=2 imbalance_ratio=2.919556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=40 tokens_per_ep_rank=[32443, 33096, 32764, 32768, 32441, 33096, 33096, 32440] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97213 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=44 tokens_per_ep_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64828 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=43 tokens_per_ep_rank=[46, 32766, 5, 19620, 65534, 95668, 15600, 32905] avg=32768.0000 max=95668 max_rank=5 min=5 min_rank=2 imbalance_ratio=2.919556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=45 tokens_per_ep_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] avg=32768.0000 max=33097 max_rank=4 min=31997 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=45997 max_load_reduction_pct=0.581549 moved_to_replicas=63938 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=45 tokens_per_ep_rank=[6, 51036, 32884, 822, 79094, 32765, 32766, 32771] avg=32768.0000 max=79094 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.413757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=43 tokens_per_ep_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] avg=32768.0000 max=33097 max_rank=3 min=31320 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62571 max_load_reduction_pct=0.654043 moved_to_replicas=95010 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=44 tokens_per_ep_rank=[65490, 9, 32767, 3, 32766, 65530, 32770, 32809] avg=32768.0000 max=65530 max_rank=5 min=3 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=43 tokens_per_ep_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] avg=32768.0000 max=33097 max_rank=3 min=31320 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62571 max_load_reduction_pct=0.654043 moved_to_replicas=95010 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=45 tokens_per_ep_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] avg=32768.0000 max=33097 max_rank=4 min=31997 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=45997 max_load_reduction_pct=0.581549 moved_to_replicas=63938 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=43 tokens_per_ep_rank=[46, 32766, 5, 19620, 65534, 95668, 15600, 32905] avg=32768.0000 max=95668 max_rank=5 min=5 min_rank=2 imbalance_ratio=2.919556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=41 tokens_per_ep_rank=[65531, 32762, 32770, 3, 3, 98294, 11, 32770] avg=32768.0000 max=98294 max_rank=5 min=3 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=45 tokens_per_ep_rank=[6, 51036, 32884, 822, 79094, 32765, 32766, 32771] avg=32768.0000 max=79094 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.413757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=46 tokens_per_ep_rank=[26021, 6, 6831, 32677, 32763, 32783, 131063, 0] avg=32768.0000 max=131063 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=44 tokens_per_ep_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64828 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=43 tokens_per_ep_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] avg=32768.0000 max=33097 max_rank=3 min=31320 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62571 max_load_reduction_pct=0.654043 moved_to_replicas=95010 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=44 tokens_per_ep_rank=[65490, 9, 32767, 3, 32766, 65530, 32770, 32809] avg=32768.0000 max=65530 max_rank=5 min=3 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=41 tokens_per_ep_rank=[33096, 32762, 32770, 32767, 32437, 33096, 32446, 32770] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=97633 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=45 tokens_per_ep_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] avg=32768.0000 max=33097 max_rank=4 min=31997 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=45997 max_load_reduction_pct=0.581549 moved_to_replicas=63938 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=46 tokens_per_ep_rank=[26021, 6, 6831, 32677, 32763, 32783, 131063, 0] avg=32768.0000 max=131063 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=44 tokens_per_ep_rank=[65490, 9, 32767, 3, 32766, 65530, 32770, 32809] avg=32768.0000 max=65530 max_rank=5 min=3 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=46 tokens_per_ep_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] avg=32768.0000 max=33096 max_rank=2 min=32195 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97967 max_load_reduction_pct=0.747480 moved_to_replicas=97967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=44 tokens_per_ep_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64828 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=45 tokens_per_ep_rank=[6, 51036, 32884, 822, 79094, 32765, 32766, 32771] avg=32768.0000 max=79094 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.413757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=46 tokens_per_ep_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] avg=32768.0000 max=33096 max_rank=2 min=32195 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97967 max_load_reduction_pct=0.747480 moved_to_replicas=97967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=44 tokens_per_ep_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64828 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=44 tokens_per_ep_rank=[65490, 9, 32767, 3, 32766, 65530, 32770, 32809] avg=32768.0000 max=65530 max_rank=5 min=3 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=42 tokens_per_ep_rank=[378, 32428, 3, 65501, 65537, 32766, 32766, 32765] avg=32768.0000 max=65537 max_rank=4 min=3 min_rank=2 imbalance_ratio=2.000031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=46 tokens_per_ep_rank=[26021, 6, 6831, 32677, 32763, 32783, 131063, 0] avg=32768.0000 max=131063 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 3, 100187, 32768, 32507, 1, 31135] avg=32768.0000 max=100187 max_rank=3 min=1 min_rank=6 imbalance_ratio=3.057465 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=45 tokens_per_ep_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] avg=32768.0000 max=33097 max_rank=4 min=31997 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=45997 max_load_reduction_pct=0.581549 moved_to_replicas=63938 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=45 tokens_per_ep_rank=[6, 51036, 32884, 822, 79094, 32765, 32766, 32771] avg=32768.0000 max=79094 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.413757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=44 tokens_per_ep_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64828 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=42 tokens_per_ep_rank=[32783, 32428, 32444, 33096, 33096, 32766, 32766, 32765] avg=32768.0000 max=33096 max_rank=3 min=32428 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32441 max_load_reduction_pct=0.495003 moved_to_replicas=64846 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=46 tokens_per_ep_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] avg=32768.0000 max=33096 max_rank=2 min=32195 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97967 max_load_reduction_pct=0.747480 moved_to_replicas=97967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 3, 100187, 32768, 32507, 1, 31135] avg=32768.0000 max=100187 max_rank=3 min=1 min_rank=6 imbalance_ratio=3.057465 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=67091 max_load_reduction_pct=0.669658 moved_to_replicas=67091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=45 tokens_per_ep_rank=[6, 51036, 32884, 822, 79094, 32765, 32766, 32771] avg=32768.0000 max=79094 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.413757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=45 tokens_per_ep_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] avg=32768.0000 max=33097 max_rank=4 min=31997 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=45997 max_load_reduction_pct=0.581549 moved_to_replicas=63938 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=46 tokens_per_ep_rank=[26021, 6, 6831, 32677, 32763, 32783, 131063, 0] avg=32768.0000 max=131063 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=67091 max_load_reduction_pct=0.669658 moved_to_replicas=67091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=45 tokens_per_ep_rank=[6, 51036, 32884, 822, 79094, 32765, 32766, 32771] avg=32768.0000 max=79094 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.413757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=45 tokens_per_ep_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] avg=32768.0000 max=33097 max_rank=4 min=31997 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=45997 max_load_reduction_pct=0.581549 moved_to_replicas=63938 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=43 tokens_per_ep_rank=[46, 32766, 5, 19620, 65534, 95668, 15600, 32905] avg=32768.0000 max=95668 max_rank=5 min=5 min_rank=2 imbalance_ratio=2.919556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 3, 100187, 32768, 32507, 1, 31135] avg=32768.0000 max=100187 max_rank=3 min=1 min_rank=6 imbalance_ratio=3.057465 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=48 tokens_per_ep_rank=[22, 65530, 32768, 32767, 3, 35557, 32767, 62730] avg=32768.0000 max=65530 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=46 tokens_per_ep_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] avg=32768.0000 max=33096 max_rank=2 min=32195 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97967 max_load_reduction_pct=0.747480 moved_to_replicas=97967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=46 tokens_per_ep_rank=[26021, 6, 6831, 32677, 32763, 32783, 131063, 0] avg=32768.0000 max=131063 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=45 tokens_per_ep_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] avg=32768.0000 max=33097 max_rank=4 min=31997 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=45997 max_load_reduction_pct=0.581549 moved_to_replicas=63938 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=48 tokens_per_ep_rank=[22, 65530, 32768, 32767, 3, 35557, 32767, 62730] avg=32768.0000 max=65530 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=67091 max_load_reduction_pct=0.669658 moved_to_replicas=67091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=43 tokens_per_ep_rank=[31320, 32766, 32769, 33097, 33096, 33096, 33095, 32905] avg=32768.0000 max=33097 max_rank=3 min=31320 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=62571 max_load_reduction_pct=0.654043 moved_to_replicas=95010 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=48 tokens_per_ep_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=32117 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64529 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=46 tokens_per_ep_rank=[26021, 6, 6831, 32677, 32763, 32783, 131063, 0] avg=32768.0000 max=131063 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=46 tokens_per_ep_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] avg=32768.0000 max=33096 max_rank=2 min=32195 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97967 max_load_reduction_pct=0.747480 moved_to_replicas=97967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 3, 100187, 32768, 32507, 1, 31135] avg=32768.0000 max=100187 max_rank=3 min=1 min_rank=6 imbalance_ratio=3.057465 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=48 tokens_per_ep_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=32117 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64529 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=46 tokens_per_ep_rank=[26021, 6, 6831, 32677, 32763, 32783, 131063, 0] avg=32768.0000 max=131063 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=46 tokens_per_ep_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] avg=32768.0000 max=33096 max_rank=2 min=32195 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97967 max_load_reduction_pct=0.747480 moved_to_replicas=97967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=67091 max_load_reduction_pct=0.669658 moved_to_replicas=67091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=48 tokens_per_ep_rank=[22, 65530, 32768, 32767, 3, 35557, 32767, 62730] avg=32768.0000 max=65530 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=44 tokens_per_ep_rank=[65490, 9, 32767, 3, 32766, 65530, 32770, 32809] avg=32768.0000 max=65530 max_rank=5 min=3 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 3, 100187, 32768, 32507, 1, 31135] avg=32768.0000 max=100187 max_rank=3 min=1 min_rank=6 imbalance_ratio=3.057465 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=46 tokens_per_ep_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] avg=32768.0000 max=33096 max_rank=2 min=32195 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97967 max_load_reduction_pct=0.747480 moved_to_replicas=97967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=48 tokens_per_ep_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=32117 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64529 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=44 tokens_per_ep_rank=[33096, 32403, 32767, 32437, 32766, 33096, 32770, 32809] avg=32768.0000 max=33096 max_rank=0 min=32403 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64828 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 3, 100187, 32768, 32507, 1, 31135] avg=32768.0000 max=100187 max_rank=3 min=1 min_rank=6 imbalance_ratio=3.057465 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=67091 max_load_reduction_pct=0.669658 moved_to_replicas=67091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=48 tokens_per_ep_rank=[22, 65530, 32768, 32767, 3, 35557, 32767, 62730] avg=32768.0000 max=65530 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 3, 100187, 32768, 32507, 1, 31135] avg=32768.0000 max=100187 max_rank=3 min=1 min_rank=6 imbalance_ratio=3.057465 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=67091 max_load_reduction_pct=0.669658 moved_to_replicas=67091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=48 tokens_per_ep_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=32117 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64529 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=45 tokens_per_ep_rank=[6, 51036, 32884, 822, 79094, 32765, 32766, 32771] avg=32768.0000 max=79094 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.413757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=48 tokens_per_ep_rank=[22, 65530, 32768, 32767, 3, 35557, 32767, 62730] avg=32768.0000 max=65530 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=67091 max_load_reduction_pct=0.669658 moved_to_replicas=67091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=45 tokens_per_ep_rank=[32770, 33094, 32884, 31997, 33097, 32765, 32766, 32771] avg=32768.0000 max=33097 max_rank=4 min=31997 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=45997 max_load_reduction_pct=0.581549 moved_to_replicas=63938 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=48 tokens_per_ep_rank=[22, 65530, 32768, 32767, 3, 35557, 32767, 62730] avg=32768.0000 max=65530 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=48 tokens_per_ep_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=32117 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64529 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=48 tokens_per_ep_rank=[22, 65530, 32768, 32767, 3, 35557, 32767, 62730] avg=32768.0000 max=65530 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=48 tokens_per_ep_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=32117 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64529 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=46 tokens_per_ep_rank=[26021, 6, 6831, 32677, 32763, 32783, 131063, 0] avg=32768.0000 max=131063 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=48 tokens_per_ep_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=32117 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64529 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=46 tokens_per_ep_rank=[32195, 32770, 33096, 32677, 32763, 32783, 33096, 32764] avg=32768.0000 max=33096 max_rank=2 min=32195 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97967 max_load_reduction_pct=0.747480 moved_to_replicas=97967 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 3, 100187, 32768, 32507, 1, 31135] avg=32768.0000 max=100187 max_rank=3 min=1 min_rank=6 imbalance_ratio=3.057465 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=47 tokens_per_ep_rank=[32777, 32766, 32767, 33096, 32768, 32507, 32765, 32698] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=67091 max_load_reduction_pct=0.669658 moved_to_replicas=67091 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=48 tokens_per_ep_rank=[22, 65530, 32768, 32767, 3, 35557, 32767, 62730] avg=32768.0000 max=65530 max_rank=1 min=3 min_rank=4 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=48 tokens_per_ep_rank=[32117, 33096, 32768, 32767, 32437, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=1 min=32117 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=64529 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=48 total_expected_recv=1570972 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380032 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=48 total_expected_recv=1576552 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377472 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=48 total_expected_recv=1571671 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=48 total_expected_recv=1576030 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375040 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=48 total_expected_recv=1570949 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380928 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=48 total_expected_recv=1571115 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379904 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=48 total_expected_recv=1571200 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380416 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=48 total_expected_recv=1574423 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376960 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:47:03] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 716.6 | throughput per GPU (TFLOP/s/GPU): 124.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.915908E+00 | loss scale: 1.0 | grad norm: 3.420 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=30820 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=30820 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31612 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31956 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31612 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=31956 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=30992 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=30992 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=30533 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=30533 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31104 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=31851 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=31851 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32227 expected_overflow=0 exact_recv_per_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32227 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=30724 expected_overflow=0 exact_recv_per_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=30724 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=31064 expected_overflow=0 exact_recv_per_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=31064 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32049 expected_overflow=0 exact_recv_per_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=31004 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=31004 actual_valid_recv=31360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=31057 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=31057 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32049 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32021 expected_overflow=0 exact_recv_per_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32021 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=31852 expected_overflow=0 exact_recv_per_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=31852 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=30861 expected_overflow=0 exact_recv_per_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33026 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=31876 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=30861 actual_valid_recv=31232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33026 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=31876 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32071 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32071 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32003 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32003 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33043 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33017 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33043 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33017 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32105 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=31088 expected_overflow=0 exact_recv_per_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=31088 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=31963 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=31963 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=31519 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=31519 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32105 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32371 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32371 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32367 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32367 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32228 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32281 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32228 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32281 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32169 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32169 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=40960 expected_recv=31854 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=40960 expected_recv=31854 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32283 expected_overflow=0 exact_recv_per_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32283 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32516 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32516 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=40960 expected_recv=31460 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=40960 expected_recv=31460 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32330 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32330 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32337 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32224 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32337 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32224 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32343 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32343 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32187 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32187 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32384 expected_overflow=0 exact_recv_per_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32384 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=40960 expected_recv=31545 expected_overflow=0 exact_recv_per_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=40960 expected_recv=31545 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=40960 expected_recv=31983 expected_overflow=0 exact_recv_per_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=40960 expected_recv=31983 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32240 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32218 expected_overflow=0 exact_recv_per_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32218 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32240 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32388 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32388 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=40960 expected_recv=31449 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=40960 expected_recv=31763 expected_overflow=0 exact_recv_per_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=40960 expected_recv=31763 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=40960 expected_recv=31449 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32089 expected_overflow=0 exact_recv_per_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32089 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32272 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32122 expected_overflow=0 exact_recv_per_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32122 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32272 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32318 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32318 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=40960 expected_recv=31981 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=40960 expected_recv=31981 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32312 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32312 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32121 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32121 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32377 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=24 calls=1 budget=40960 expected_recv=31780 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32377 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=24 calls=1 budget=40960 expected_recv=31780 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=26 calls=1 budget=40960 expected_recv=31768 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=26 calls=1 budget=40960 expected_recv=31768 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32042 expected_overflow=0 exact_recv_per_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32042 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32132 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32132 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32325 expected_overflow=0 exact_recv_per_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32325 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32119 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=28 calls=1 budget=40960 expected_recv=31803 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32119 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=28 calls=1 budget=40960 expected_recv=31803 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32104 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32104 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=25 calls=1 budget=40960 expected_recv=31750 expected_overflow=0 exact_recv_per_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=27 calls=1 budget=40960 expected_recv=31477 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=25 calls=1 budget=40960 expected_recv=31750 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=27 calls=1 budget=40960 expected_recv=31477 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=26 calls=1 budget=40960 expected_recv=31742 expected_overflow=0 exact_recv_per_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=26 calls=1 budget=40960 expected_recv=31742 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32557 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32557 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32034 expected_overflow=0 exact_recv_per_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32289 expected_overflow=0 exact_recv_per_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32034 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32289 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=28 calls=1 budget=40960 expected_recv=31672 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=28 calls=1 budget=40960 expected_recv=31672 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32143 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32143 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32376 expected_overflow=0 exact_recv_per_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32376 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32109 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32109 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32477 expected_overflow=0 exact_recv_per_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32369 expected_overflow=0 exact_recv_per_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32369 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32477 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32154 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32172 expected_overflow=0 exact_recv_per_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32172 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32154 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=35 calls=1 budget=40960 expected_recv=31791 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=35 calls=1 budget=40960 expected_recv=31791 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32306 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32306 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=35 calls=1 budget=40960 expected_recv=31772 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=35 calls=1 budget=40960 expected_recv=31772 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32452 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32452 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32415 expected_overflow=0 exact_recv_per_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32415 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32302 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32302 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32180 expected_overflow=0 exact_recv_per_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32180 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32379 expected_overflow=0 exact_recv_per_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32379 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32462 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=40 calls=1 budget=40960 expected_recv=31730 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32024 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=40 calls=1 budget=40960 expected_recv=31730 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32024 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32462 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=41 calls=1 budget=40960 expected_recv=31756 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=41 calls=1 budget=40960 expected_recv=31756 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=42 calls=1 budget=40960 expected_recv=31226 expected_overflow=0 exact_recv_per_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=42 calls=1 budget=40960 expected_recv=31226 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32027 expected_overflow=0 exact_recv_per_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32945 expected_overflow=0 exact_recv_per_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32370 expected_overflow=0 exact_recv_per_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32027 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32945 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32370 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=42 calls=1 budget=40960 expected_recv=31780 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=42 calls=1 budget=40960 expected_recv=31780 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32227 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32227 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32115 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32115 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=45 calls=1 budget=40960 expected_recv=31483 expected_overflow=0 exact_recv_per_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=45 calls=1 budget=40960 expected_recv=31483 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=43 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32092 expected_overflow=0 exact_recv_per_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32092 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32427 expected_overflow=0 exact_recv_per_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32427 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=47 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=46 calls=1 budget=40960 expected_recv=31906 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=47 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=46 calls=1 budget=40960 expected_recv=31906 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[24045, 23935, 36596, 38776, 42379, 28342, 30693, 37378] avg=32768.0000 max=42379 max_rank=4 min=23935 min_rank=1 imbalance_ratio=1.293304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 33096, 32624, 30944, 33096] avg=32768.0000 max=33097 max_rank=0 min=30944 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=9282 max_load_reduction_pct=0.219024 moved_to_replicas=22745 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[80660, 6267, 469, 29456, 19801, 37367, 67207, 20917] avg=32768.0000 max=80660 max_rank=0 min=469 min_rank=2 imbalance_ratio=2.461548 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33096, 31956, 33096, 31612, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31612 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=47564 max_load_reduction_pct=0.589685 moved_to_replicas=85946 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[71668, 12069, 12184, 30992, 71365, 15520, 23766, 24580] avg=32768.0000 max=71668 max_rank=0 min=12069 min_rank=1 imbalance_ratio=2.187134 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 30992, 33095, 33096, 33096, 32577] avg=32768.0000 max=33096 max_rank=0 min=30992 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=38572 max_load_reduction_pct=0.538204 moved_to_replicas=76841 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[20, 32951, 32731, 44899, 625, 31851, 59551, 59516] avg=32768.0000 max=59551 max_rank=6 min=20 min_rank=0 imbalance_ratio=1.817352 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[33097, 32951, 32731, 33096, 32227, 31851, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31851 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=64678 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[92874, 360, 1466, 42485, 65596, 6942, 27763, 24658] avg=32768.0000 max=92874 max_rank=0 min=360 min_rank=1 imbalance_ratio=2.834290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[33096, 33096, 32564, 33096, 33096, 33096, 33096, 31004] avg=32768.0000 max=33096 max_rank=0 min=31004 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=59778 max_load_reduction_pct=0.643646 moved_to_replicas=101667 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[22833, 23240, 33224, 43326, 38137, 64005, 32512, 4867] avg=32768.0000 max=64005 max_rank=5 min=4867 min_rank=7 imbalance_ratio=1.953278 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[31057, 33096, 33096, 33096, 33095, 33096, 32512, 33096] avg=32768.0000 max=33096 max_rank=1 min=31057 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30909 max_load_reduction_pct=0.482915 moved_to_replicas=46308 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[35774, 30284, 962, 32841, 12498, 45627, 49480, 54678] avg=32768.0000 max=54678 max_rank=7 min=962 min_rank=2 imbalance_ratio=1.668640 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[33096, 32962, 33096, 32841, 30861, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30861 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21582 max_load_reduction_pct=0.394711 moved_to_replicas=53175 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[33026, 7, 84801, 31876, 43391, 2881, 1, 66161] avg=32768.0000 max=84801 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.587921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[33026, 32786, 33096, 31876, 33097, 33096, 32071, 33096] avg=32768.0000 max=33097 max_rank=4 min=31876 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51704 max_load_reduction_pct=0.609710 moved_to_replicas=95065 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32776, 65543, 32756, 2, 32925, 58, 33043, 65041] avg=32768.0000 max=65543 max_rank=1 min=2 min_rank=3 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32776, 33096, 32756, 32449, 32925, 32003, 33043, 33096] avg=32768.0000 max=33096 max_rank=1 min=32003 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=64392 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[2275, 64995, 64840, 1273, 645, 32105, 32769, 63242] avg=32768.0000 max=64995 max_rank=1 min=645 min_rank=4 imbalance_ratio=1.983490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[32421, 33096, 33096, 33017, 32544, 32105, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=32105 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31899 max_load_reduction_pct=0.490792 moved_to_replicas=93789 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[39814, 58845, 32764, 32542, 33625, 31842, 32706, 6] avg=32768.0000 max=58845 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.795807 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[33097, 33096, 32764, 32542, 33096, 32371, 32706, 32472] avg=32768.0000 max=33097 max_rank=0 min=32371 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25748 max_load_reduction_pct=0.437556 moved_to_replicas=32996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[24012, 2524, 65374, 32735, 36675, 41313, 59508, 3] avg=32768.0000 max=65374 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.995056 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[32228, 32516, 33096, 32735, 33096, 33097, 33095, 32281] avg=32768.0000 max=33097 max_rank=5 min=32228 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32277 max_load_reduction_pct=0.493728 moved_to_replicas=70486 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32765, 73459, 82548, 25, 2, 453, 32768, 40124] avg=32768.0000 max=82548 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.519165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32765, 33096, 33096, 31460, 32767, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31460 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=49452 max_load_reduction_pct=0.599070 moved_to_replicas=96843 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32330, 65521, 65080, 33240, 32295, 726, 32753, 199] avg=32768.0000 max=65521 max_rank=1 min=199 min_rank=7 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32330, 33096, 33096, 33096, 32439, 32710, 32753, 32624] avg=32768.0000 max=33096 max_rank=1 min=32330 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=64553 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32774, 65520, 32224, 3, 46645, 28405, 56572, 1] avg=32768.0000 max=65520 max_rank=1 min=1 min_rank=7 imbalance_ratio=1.999512 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32774, 33096, 32224, 33097, 33094, 32337, 33097, 32425] avg=32768.0000 max=33097 max_rank=3 min=32224 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32423 max_load_reduction_pct=0.494857 moved_to_replicas=69449 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[27718, 33018, 32958, 32767, 32449, 32804, 112, 70318] avg=32768.0000 max=70318 max_rank=7 min=112 min_rank=6 imbalance_ratio=2.145935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[32343, 33018, 32958, 32767, 32449, 32804, 32709, 33096] avg=32768.0000 max=33096 max_rank=7 min=32343 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37222 max_load_reduction_pct=0.529338 moved_to_replicas=37222 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[3, 32240, 32766, 131030, 9, 65658, 151, 287] avg=32768.0000 max=131030 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[32768, 32240, 32766, 33096, 32774, 33096, 32555, 32849] avg=32768.0000 max=33096 max_rank=3 min=32240 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97934 max_load_reduction_pct=0.747417 moved_to_replicas=130496 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[65527, 402, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=65527 max_rank=0 min=402 min_rank=1 imbalance_ratio=1.999725 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[33096, 32833, 32703, 32766, 32820, 32771, 32767, 32388] avg=32768.0000 max=33096 max_rank=0 min=32388 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=32431 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[20, 0, 62561, 59329, 32775, 3933, 32776, 70750] avg=32768.0000 max=70750 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.159119 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[31449, 32763, 33096, 33094, 32775, 33096, 32776, 33095] avg=32768.0000 max=33096 max_rank=2 min=31449 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=37654 max_load_reduction_pct=0.532212 moved_to_replicas=93352 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33146, 711, 64607, 32770, 0, 32318, 65815] avg=32768.0000 max=65815 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.008514 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[32777, 33096, 32272, 33096, 32770, 32719, 32318, 33096] avg=32768.0000 max=33096 max_rank=1 min=32272 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32719 max_load_reduction_pct=0.497136 moved_to_replicas=64280 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[119, 3, 5, 5, 76163, 120301, 32806, 32742] avg=32768.0000 max=120301 max_rank=5 min=3 min_rank=1 imbalance_ratio=3.671295 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[32884, 32768, 32770, 31981, 33096, 33097, 32806, 32742] avg=32768.0000 max=33097 max_rank=5 min=31981 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=87204 max_load_reduction_pct=0.724882 moved_to_replicas=130272 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[98169, 4, 2, 32726, 65716, 32774, 12, 32741] avg=32768.0000 max=98169 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.995880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[33096, 32312, 32767, 32726, 33096, 32774, 32632, 32741] avg=32768.0000 max=33096 max_rank=0 min=32312 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.662867 moved_to_replicas=97693 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[32757, 65571, 65705, 32464, 65533, 107, 1, 6] avg=32768.0000 max=65705 max_rank=2 min=1 min_rank=6 imbalance_ratio=2.005157 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[32757, 33096, 33096, 32464, 33096, 32544, 32610, 32481] avg=32768.0000 max=33096 max_rank=1 min=32464 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32609 max_load_reduction_pct=0.496294 moved_to_replicas=97521 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[3, 31772, 4, 65521, 32783, 33287, 65520, 33254] avg=32768.0000 max=65521 max_rank=3 min=3 min_rank=0 imbalance_ratio=1.999542 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[32428, 32121, 32428, 33096, 32783, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32121 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=65198 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=25 tokens_per_ep_rank=[45786, 32750, 97517, 32793, 0, 2, 32768, 20528] avg=32768.0000 max=97517 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.975983 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=25 tokens_per_ep_rank=[33097, 32750, 33096, 32793, 32765, 31780, 32768, 33095] avg=32768.0000 max=33097 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=64420 max_load_reduction_pct=0.660603 moved_to_replicas=77111 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=26 tokens_per_ep_rank=[603, 64870, 32766, 65534, 32769, 4, 65415, 183] avg=32768.0000 max=65534 max_rank=3 min=4 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=26 tokens_per_ep_rank=[32377, 33096, 32766, 33096, 32769, 32442, 33096, 32502] avg=32768.0000 max=33096 max_rank=1 min=32377 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96531 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 0, 84093, 18939, 60784, 3] avg=32768.0000 max=84093 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.566315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=27 tokens_per_ep_rank=[32772, 32767, 32786, 32764, 33096, 31768, 33096, 33095] avg=32768.0000 max=33096 max_rank=4 min=31768 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=50997 max_load_reduction_pct=0.606436 moved_to_replicas=78685 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=28 tokens_per_ep_rank=[5, 65205, 30773, 35124, 98257, 10, 2, 32768] avg=32768.0000 max=98257 max_rank=4 min=2 min_rank=6 imbalance_ratio=2.998566 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=28 tokens_per_ep_rank=[32401, 33096, 32801, 33096, 33096, 32119, 32767, 32768] avg=32768.0000 max=33096 max_rank=1 min=32119 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65161 max_load_reduction_pct=0.663169 moved_to_replicas=99298 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=29 tokens_per_ep_rank=[9571, 32759, 65535, 32777, 35013, 32773, 32745, 20971] avg=32768.0000 max=65535 max_rank=2 min=9571 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=29 tokens_per_ep_rank=[33095, 32759, 33096, 32777, 33096, 32773, 32745, 31803] avg=32768.0000 max=33096 max_rank=2 min=31803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=34356 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=30 tokens_per_ep_rank=[32104, 65540, 0, 32771, 32741, 2, 66214, 32772] avg=32768.0000 max=66214 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.020691 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=30 tokens_per_ep_rank=[32104, 33096, 32765, 32771, 32741, 32799, 33096, 32772] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33118 max_load_reduction_pct=0.500166 moved_to_replicas=65562 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=31 tokens_per_ep_rank=[6, 32594, 65712, 32766, 32768, 32766, 0, 65532] avg=32768.0000 max=65712 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=31 tokens_per_ep_rank=[32442, 32594, 33096, 32766, 32768, 32766, 32616, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32616 max_load_reduction_pct=0.496348 moved_to_replicas=65052 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=32 tokens_per_ep_rank=[2, 32735, 3, 42126, 88975, 32771, 32766, 32766] avg=32768.0000 max=88975 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.715302 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=32 tokens_per_ep_rank=[32767, 32735, 32143, 33098, 33098, 32771, 32766, 32766] avg=32768.0000 max=33098 max_rank=3 min=32143 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=55877 max_load_reduction_pct=0.628008 moved_to_replicas=64909 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=33 tokens_per_ep_rank=[5, 17667, 66872, 2, 32766, 0, 98304, 46528] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=33 tokens_per_ep_rank=[32770, 32109, 33096, 32445, 32766, 32765, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32109 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=65207 max_load_reduction_pct=0.663320 moved_to_replicas=112416 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=34 tokens_per_ep_rank=[10223, 32743, 98301, 2, 32727, 18358, 25, 69765] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=3 imbalance_ratio=2.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=34 tokens_per_ep_rank=[33096, 32743, 33096, 32767, 32727, 32154, 32465, 33096] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=101874 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=35 tokens_per_ep_rank=[32708, 3, 32772, 98452, 8, 114, 65531, 32556] avg=32768.0000 max=98452 max_rank=3 min=3 min_rank=1 imbalance_ratio=3.004517 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=35 tokens_per_ep_rank=[32708, 32768, 32772, 33096, 32599, 32549, 33096, 32556] avg=32768.0000 max=33096 max_rank=3 min=32549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65356 max_load_reduction_pct=0.663836 moved_to_replicas=97791 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=36 tokens_per_ep_rank=[65534, 19262, 0, 32763, 7785, 65526, 32768, 38506] avg=32768.0000 max=65534 max_rank=0 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=36 tokens_per_ep_rank=[33096, 31791, 32438, 32763, 33095, 33096, 32768, 33097] avg=32768.0000 max=33097 max_rank=7 min=31791 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32437 max_load_reduction_pct=0.494964 moved_to_replicas=70278 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 65402, 0, 32767, 32901] avg=32768.0000 max=65402 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.995911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=37 tokens_per_ep_rank=[32768, 32770, 32770, 32766, 33096, 32306, 32767, 32901] avg=32768.0000 max=33096 max_rank=4 min=32306 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32306 max_load_reduction_pct=0.493960 moved_to_replicas=32306 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=38 tokens_per_ep_rank=[32768, 0, 98305, 10, 32773, 65519, 3, 32766] avg=32768.0000 max=98305 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.000031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=38 tokens_per_ep_rank=[32768, 32765, 33096, 32433, 32773, 33096, 32447, 32766] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65209 max_load_reduction_pct=0.663334 moved_to_replicas=97632 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=39 tokens_per_ep_rank=[0, 2, 65533, 4, 32763, 65493, 65534, 32815] avg=32768.0000 max=65534 max_rank=6 min=0 min_rank=0 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=39 tokens_per_ep_rank=[32438, 32439, 33096, 32401, 32763, 33096, 33096, 32815] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=97272 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=40 tokens_per_ep_rank=[65826, 32940, 0, 65535, 32770, 32302, 32769, 2] avg=32768.0000 max=65826 max_rank=0 min=0 min_rank=2 imbalance_ratio=2.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=40 tokens_per_ep_rank=[33096, 32940, 32730, 33096, 32770, 32302, 32769, 32441] avg=32768.0000 max=33096 max_rank=0 min=32302 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32730 max_load_reduction_pct=0.497220 moved_to_replicas=65169 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=41 tokens_per_ep_rank=[55748, 2, 65588, 23758, 3712, 47798, 6, 65532] avg=32768.0000 max=65588 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=41 tokens_per_ep_rank=[33095, 32494, 33096, 33094, 31730, 33097, 32442, 33096] avg=32768.0000 max=33097 max_rank=5 min=31730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=102282 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 280, 32771, 65535, 62400, 35535, 0] avg=32768.0000 max=65535 max_rank=4 min=0 min_rank=7 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=42 tokens_per_ep_rank=[32856, 32767, 32024, 32771, 33096, 33096, 33095, 32439] avg=32768.0000 max=33096 max_rank=4 min=32024 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=64182 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=43 tokens_per_ep_rank=[67079, 2, 6, 4, 32770, 96402, 33113, 32768] avg=32768.0000 max=96402 max_rank=5 min=2 min_rank=1 imbalance_ratio=2.941956 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=43 tokens_per_ep_rank=[33096, 32767, 32771, 31780, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63306 max_load_reduction_pct=0.656688 moved_to_replicas=97306 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=44 tokens_per_ep_rank=[32768, 4, 65515, 65643, 6167, 32785, 59158, 104] avg=32768.0000 max=65643 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=44 tokens_per_ep_rank=[32768, 32551, 33096, 33096, 32227, 32785, 33098, 32523] avg=32768.0000 max=33098 max_rank=6 min=32227 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32545 max_load_reduction_pct=0.495788 moved_to_replicas=91028 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 2, 65533, 32772, 32719, 32805] avg=32768.0000 max=65533 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=45 tokens_per_ep_rank=[32770, 32775, 32768, 32439, 33096, 32772, 32719, 32805] avg=32768.0000 max=33096 max_rank=4 min=32439 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32437 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=46 tokens_per_ep_rank=[32759, 5, 10, 32766, 32767, 12523, 131070, 20244] avg=32768.0000 max=131070 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=46 tokens_per_ep_rank=[32759, 32770, 32775, 32766, 32767, 33096, 33096, 32115] avg=32768.0000 max=33096 max_rank=5 min=32115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97974 max_load_reduction_pct=0.747494 moved_to_replicas=97974 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=47 tokens_per_ep_rank=[5, 2044, 32768, 66081, 32766, 62958, 65522, 0] avg=32768.0000 max=66081 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.016632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=47 tokens_per_ep_rank=[32651, 31906, 32768, 33096, 32766, 33096, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=31906 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32985 max_load_reduction_pct=0.499160 moved_to_replicas=95273 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=48 tokens_per_ep_rank=[29195, 32766, 65534, 2, 4, 86636, 0, 48007] avg=32768.0000 max=86636 max_rank=5 min=0 min_rank=6 imbalance_ratio=2.643921 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=48 tokens_per_ep_rank=[31787, 32766, 33096, 33095, 32442, 33096, 32765, 33097] avg=32768.0000 max=33097 max_rank=7 min=31787 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=53539 max_load_reduction_pct=0.617976 moved_to_replicas=100889 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=48 total_expected_recv=1573105 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376960 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=48 total_expected_recv=1571483 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377216 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=48 total_expected_recv=1570227 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379264 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=48 total_expected_recv=1570252 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380544 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=48 total_expected_recv=1574195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375936 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=48 total_expected_recv=1576891 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372864 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=48 total_expected_recv=1572392 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378240 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=48 total_expected_recv=1574367 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378368 peak_demand=33098 required_capacity=1.010071 - [2026-08-26 15:01:25] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 722.0 | throughput per GPU (TFLOP/s/GPU): 123.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.820069E+00 | loss scale: 1.0 | grad norm: 2.904 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32072 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31538 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32072 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31538 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31232 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[22438, 26452, 32749, 45534, 42855, 25189, 27337, 39590] avg=32768.0000 max=45534 max_rank=3 min=22438 min_rank=0 imbalance_ratio=1.389587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[22438, 26452, 32749, 45534, 42855, 25189, 27337, 39590] avg=32768.0000 max=45534 max_rank=3 min=22438 min_rank=0 imbalance_ratio=1.389587 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[22438, 26452, 32749, 45534, 42855, 25189, 27337, 39590] avg=32768.0000 max=45534 max_rank=3 min=22438 min_rank=0 imbalance_ratio=1.389587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[22438, 26452, 32749, 45534, 42855, 25189, 27337, 39590] avg=32768.0000 max=45534 max_rank=3 min=22438 min_rank=0 imbalance_ratio=1.389587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[22438, 26452, 32749, 45534, 42855, 25189, 27337, 39590] avg=32768.0000 max=45534 max_rank=3 min=22438 min_rank=0 imbalance_ratio=1.389587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] avg=32768.0000 max=33097 max_rank=5 min=30820 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=12437 max_load_reduction_pct=0.273137 moved_to_replicas=28691 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] avg=32768.0000 max=33097 max_rank=5 min=30820 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=12437 max_load_reduction_pct=0.273137 moved_to_replicas=28691 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] avg=32768.0000 max=33097 max_rank=5 min=30820 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=12437 max_load_reduction_pct=0.273137 moved_to_replicas=28691 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[22438, 26452, 32749, 45534, 42855, 25189, 27337, 39590] avg=32768.0000 max=45534 max_rank=3 min=22438 min_rank=0 imbalance_ratio=1.389587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] avg=32768.0000 max=33097 max_rank=5 min=30820 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=12437 max_load_reduction_pct=0.273137 moved_to_replicas=28691 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] avg=32768.0000 max=33097 max_rank=5 min=30820 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=12437 max_load_reduction_pct=0.273137 moved_to_replicas=28691 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[22438, 26452, 32749, 45534, 42855, 25189, 27337, 39590] avg=32768.0000 max=45534 max_rank=3 min=22438 min_rank=0 imbalance_ratio=1.389587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[79842, 5723, 5489, 53340, 8198, 41349, 33035, 35168] avg=32768.0000 max=79842 max_rank=0 min=5489 min_rank=2 imbalance_ratio=2.436584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[79842, 5723, 5489, 53340, 8198, 41349, 33035, 35168] avg=32768.0000 max=79842 max_rank=0 min=5489 min_rank=2 imbalance_ratio=2.436584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] avg=32768.0000 max=33097 max_rank=5 min=30820 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=12437 max_load_reduction_pct=0.273137 moved_to_replicas=28691 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[79842, 5723, 5489, 53340, 8198, 41349, 33035, 35168] avg=32768.0000 max=79842 max_rank=0 min=5489 min_rank=2 imbalance_ratio=2.436584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] avg=32768.0000 max=33097 max_rank=5 min=30820 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=12437 max_load_reduction_pct=0.273137 moved_to_replicas=28691 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[79842, 5723, 5489, 53340, 8198, 41349, 33035, 35168] avg=32768.0000 max=79842 max_rank=0 min=5489 min_rank=2 imbalance_ratio=2.436584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[79842, 5723, 5489, 53340, 8198, 41349, 33035, 35168] avg=32768.0000 max=79842 max_rank=0 min=5489 min_rank=2 imbalance_ratio=2.436584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32048 expected_overflow=0 exact_recv_per_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] avg=32768.0000 max=33096 max_rank=0 min=30533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46746 max_load_reduction_pct=0.585481 moved_to_replicas=77315 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] avg=32768.0000 max=33096 max_rank=0 min=30533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46746 max_load_reduction_pct=0.585481 moved_to_replicas=77315 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32048 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] avg=32768.0000 max=33096 max_rank=0 min=30533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46746 max_load_reduction_pct=0.585481 moved_to_replicas=77315 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] avg=32768.0000 max=33096 max_rank=0 min=30533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46746 max_load_reduction_pct=0.585481 moved_to_replicas=77315 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] avg=32768.0000 max=33096 max_rank=0 min=30533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46746 max_load_reduction_pct=0.585481 moved_to_replicas=77315 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[79842, 5723, 5489, 53340, 8198, 41349, 33035, 35168] avg=32768.0000 max=79842 max_rank=0 min=5489 min_rank=2 imbalance_ratio=2.436584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[79842, 5723, 5489, 53340, 8198, 41349, 33035, 35168] avg=32768.0000 max=79842 max_rank=0 min=5489 min_rank=2 imbalance_ratio=2.436584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[89609, 29902, 20148, 3344, 25287, 69872, 11686, 12296] avg=32768.0000 max=89609 max_rank=0 min=3344 min_rank=3 imbalance_ratio=2.734650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[89609, 29902, 20148, 3344, 25287, 69872, 11686, 12296] avg=32768.0000 max=89609 max_rank=0 min=3344 min_rank=3 imbalance_ratio=2.734650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[89609, 29902, 20148, 3344, 25287, 69872, 11686, 12296] avg=32768.0000 max=89609 max_rank=0 min=3344 min_rank=3 imbalance_ratio=2.734650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] avg=32768.0000 max=33096 max_rank=0 min=30533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46746 max_load_reduction_pct=0.585481 moved_to_replicas=77315 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] avg=32768.0000 max=33096 max_rank=0 min=30533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46746 max_load_reduction_pct=0.585481 moved_to_replicas=77315 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[89609, 29902, 20148, 3344, 25287, 69872, 11686, 12296] avg=32768.0000 max=89609 max_rank=0 min=3344 min_rank=3 imbalance_ratio=2.734650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[89609, 29902, 20148, 3344, 25287, 69872, 11686, 12296] avg=32768.0000 max=89609 max_rank=0 min=3344 min_rank=3 imbalance_ratio=2.734650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[22438, 26452, 32749, 45534, 42855, 25189, 27337, 39590] avg=32768.0000 max=45534 max_rank=3 min=22438 min_rank=0 imbalance_ratio=1.389587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=56513 max_load_reduction_pct=0.630662 moved_to_replicas=93289 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=56513 max_load_reduction_pct=0.630662 moved_to_replicas=93289 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=56513 max_load_reduction_pct=0.630662 moved_to_replicas=93289 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=56513 max_load_reduction_pct=0.630662 moved_to_replicas=93289 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=56513 max_load_reduction_pct=0.630662 moved_to_replicas=93289 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[33095, 33095, 32749, 33096, 33096, 33097, 30820, 33096] avg=32768.0000 max=33097 max_rank=5 min=30820 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=12437 max_load_reduction_pct=0.273137 moved_to_replicas=28691 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[89609, 29902, 20148, 3344, 25287, 69872, 11686, 12296] avg=32768.0000 max=89609 max_rank=0 min=3344 min_rank=3 imbalance_ratio=2.734650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[89609, 29902, 20148, 3344, 25287, 69872, 11686, 12296] avg=32768.0000 max=89609 max_rank=0 min=3344 min_rank=3 imbalance_ratio=2.734650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[125, 105, 45766, 51199, 7426, 72938, 50972, 33613] avg=32768.0000 max=72938 max_rank=5 min=105 min_rank=1 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[125, 105, 45766, 51199, 7426, 72938, 50972, 33613] avg=32768.0000 max=72938 max_rank=5 min=105 min_rank=1 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[125, 105, 45766, 51199, 7426, 72938, 50972, 33613] avg=32768.0000 max=72938 max_rank=5 min=105 min_rank=1 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=56513 max_load_reduction_pct=0.630662 moved_to_replicas=93289 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[125, 105, 45766, 51199, 7426, 72938, 50972, 33613] avg=32768.0000 max=72938 max_rank=5 min=105 min_rank=1 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=56513 max_load_reduction_pct=0.630662 moved_to_replicas=93289 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[125, 105, 45766, 51199, 7426, 72938, 50972, 33613] avg=32768.0000 max=72938 max_rank=5 min=105 min_rank=1 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[79842, 5723, 5489, 53340, 8198, 41349, 33035, 35168] avg=32768.0000 max=79842 max_rank=0 min=5489 min_rank=2 imbalance_ratio=2.436584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30724 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39841 max_load_reduction_pct=0.546231 moved_to_replicas=89008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30724 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39841 max_load_reduction_pct=0.546231 moved_to_replicas=89008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30724 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39841 max_load_reduction_pct=0.546231 moved_to_replicas=89008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30724 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39841 max_load_reduction_pct=0.546231 moved_to_replicas=89008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30724 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39841 max_load_reduction_pct=0.546231 moved_to_replicas=89008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[125, 105, 45766, 51199, 7426, 72938, 50972, 33613] avg=32768.0000 max=72938 max_rank=5 min=105 min_rank=1 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30533, 33096, 33035, 33096] avg=32768.0000 max=33096 max_rank=0 min=30533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46746 max_load_reduction_pct=0.585481 moved_to_replicas=77315 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[125, 105, 45766, 51199, 7426, 72938, 50972, 33613] avg=32768.0000 max=72938 max_rank=5 min=105 min_rank=1 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[29744, 32813, 36043, 63183, 44878, 22634, 6, 32843] avg=32768.0000 max=63183 max_rank=3 min=6 min_rank=6 imbalance_ratio=1.928192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[29744, 32813, 36043, 63183, 44878, 22634, 6, 32843] avg=32768.0000 max=63183 max_rank=3 min=6 min_rank=6 imbalance_ratio=1.928192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[29744, 32813, 36043, 63183, 44878, 22634, 6, 32843] avg=32768.0000 max=63183 max_rank=3 min=6 min_rank=6 imbalance_ratio=1.928192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30724 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39841 max_load_reduction_pct=0.546231 moved_to_replicas=89008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[29744, 32813, 36043, 63183, 44878, 22634, 6, 32843] avg=32768.0000 max=63183 max_rank=3 min=6 min_rank=6 imbalance_ratio=1.928192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30724 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39841 max_load_reduction_pct=0.546231 moved_to_replicas=89008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[29744, 32813, 36043, 63183, 44878, 22634, 6, 32843] avg=32768.0000 max=63183 max_rank=3 min=6 min_rank=6 imbalance_ratio=1.928192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] avg=32768.0000 max=33096 max_rank=2 min=31064 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30087 max_load_reduction_pct=0.476188 moved_to_replicas=44816 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] avg=32768.0000 max=33096 max_rank=2 min=31064 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30087 max_load_reduction_pct=0.476188 moved_to_replicas=44816 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[89609, 29902, 20148, 3344, 25287, 69872, 11686, 12296] avg=32768.0000 max=89609 max_rank=0 min=3344 min_rank=3 imbalance_ratio=2.734650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] avg=32768.0000 max=33096 max_rank=2 min=31064 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30087 max_load_reduction_pct=0.476188 moved_to_replicas=44816 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] avg=32768.0000 max=33096 max_rank=2 min=31064 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30087 max_load_reduction_pct=0.476188 moved_to_replicas=44816 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] avg=32768.0000 max=33096 max_rank=2 min=31064 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30087 max_load_reduction_pct=0.476188 moved_to_replicas=44816 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[29744, 32813, 36043, 63183, 44878, 22634, 6, 32843] avg=32768.0000 max=63183 max_rank=3 min=6 min_rank=6 imbalance_ratio=1.928192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=56513 max_load_reduction_pct=0.630662 moved_to_replicas=93289 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[29744, 32813, 36043, 63183, 44878, 22634, 6, 32843] avg=32768.0000 max=63183 max_rank=3 min=6 min_rank=6 imbalance_ratio=1.928192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[90518, 24876, 32021, 15849, 32850, 112, 79, 65839] avg=32768.0000 max=90518 max_rank=0 min=79 min_rank=6 imbalance_ratio=2.762390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[90518, 24876, 32021, 15849, 32850, 112, 79, 65839] avg=32768.0000 max=90518 max_rank=0 min=79 min_rank=6 imbalance_ratio=2.762390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[90518, 24876, 32021, 15849, 32850, 112, 79, 65839] avg=32768.0000 max=90518 max_rank=0 min=79 min_rank=6 imbalance_ratio=2.762390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] avg=32768.0000 max=33096 max_rank=2 min=31064 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30087 max_load_reduction_pct=0.476188 moved_to_replicas=44816 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[90518, 24876, 32021, 15849, 32850, 112, 79, 65839] avg=32768.0000 max=90518 max_rank=0 min=79 min_rank=6 imbalance_ratio=2.762390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[90518, 24876, 32021, 15849, 32850, 112, 79, 65839] avg=32768.0000 max=90518 max_rank=0 min=79 min_rank=6 imbalance_ratio=2.762390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] avg=32768.0000 max=33096 max_rank=0 min=32021 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57422 max_load_reduction_pct=0.634371 moved_to_replicas=90165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] avg=32768.0000 max=33096 max_rank=0 min=32021 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57422 max_load_reduction_pct=0.634371 moved_to_replicas=90165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] avg=32768.0000 max=33096 max_rank=2 min=31064 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30087 max_load_reduction_pct=0.476188 moved_to_replicas=44816 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[125, 105, 45766, 51199, 7426, 72938, 50972, 33613] avg=32768.0000 max=72938 max_rank=5 min=105 min_rank=1 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] avg=32768.0000 max=33096 max_rank=0 min=32021 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57422 max_load_reduction_pct=0.634371 moved_to_replicas=90165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] avg=32768.0000 max=33096 max_rank=0 min=32021 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57422 max_load_reduction_pct=0.634371 moved_to_replicas=90165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] avg=32768.0000 max=33096 max_rank=0 min=32021 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57422 max_load_reduction_pct=0.634371 moved_to_replicas=90165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[90518, 24876, 32021, 15849, 32850, 112, 79, 65839] avg=32768.0000 max=90518 max_rank=0 min=79 min_rank=6 imbalance_ratio=2.762390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[30724, 32842, 33097, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30724 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39841 max_load_reduction_pct=0.546231 moved_to_replicas=89008 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[18, 65604, 65373, 321, 32783, 35952, 2221, 59872] avg=32768.0000 max=65604 max_rank=1 min=18 min_rank=0 imbalance_ratio=2.002075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[18, 65604, 65373, 321, 32783, 35952, 2221, 59872] avg=32768.0000 max=65604 max_rank=1 min=18 min_rank=0 imbalance_ratio=2.002075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[18, 65604, 65373, 321, 32783, 35952, 2221, 59872] avg=32768.0000 max=65604 max_rank=1 min=18 min_rank=0 imbalance_ratio=2.002075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[90518, 24876, 32021, 15849, 32850, 112, 79, 65839] avg=32768.0000 max=90518 max_rank=0 min=79 min_rank=6 imbalance_ratio=2.762390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[18, 65604, 65373, 321, 32783, 35952, 2221, 59872] avg=32768.0000 max=65604 max_rank=1 min=18 min_rank=0 imbalance_ratio=2.002075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] avg=32768.0000 max=33096 max_rank=0 min=32021 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57422 max_load_reduction_pct=0.634371 moved_to_replicas=90165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] avg=32768.0000 max=33097 max_rank=7 min=31852 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32507 max_load_reduction_pct=0.495503 moved_to_replicas=94417 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] avg=32768.0000 max=33097 max_rank=7 min=31852 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32507 max_load_reduction_pct=0.495503 moved_to_replicas=94417 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[18, 65604, 65373, 321, 32783, 35952, 2221, 59872] avg=32768.0000 max=65604 max_rank=1 min=18 min_rank=0 imbalance_ratio=2.002075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] avg=32768.0000 max=33097 max_rank=7 min=31852 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32507 max_load_reduction_pct=0.495503 moved_to_replicas=94417 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] avg=32768.0000 max=33096 max_rank=0 min=32021 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57422 max_load_reduction_pct=0.634371 moved_to_replicas=90165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[29744, 32813, 36043, 63183, 44878, 22634, 6, 32843] avg=32768.0000 max=63183 max_rank=3 min=6 min_rank=6 imbalance_ratio=1.928192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] avg=32768.0000 max=33097 max_rank=7 min=31852 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32507 max_load_reduction_pct=0.495503 moved_to_replicas=94417 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[18, 65604, 65373, 321, 32783, 35952, 2221, 59872] avg=32768.0000 max=65604 max_rank=1 min=18 min_rank=0 imbalance_ratio=2.002075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] avg=32768.0000 max=33097 max_rank=7 min=31852 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32507 max_load_reduction_pct=0.495503 moved_to_replicas=94417 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[31064, 32813, 33096, 33096, 33096, 33096, 33040, 32843] avg=32768.0000 max=33096 max_rank=2 min=31064 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30087 max_load_reduction_pct=0.476188 moved_to_replicas=44816 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[72100, 13, 120463, 2, 33102, 32815, 2133, 1516] avg=32768.0000 max=120463 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.676239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[72100, 13, 120463, 2, 33102, 32815, 2133, 1516] avg=32768.0000 max=120463 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.676239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[72100, 13, 120463, 2, 33102, 32815, 2133, 1516] avg=32768.0000 max=120463 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.676239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[18, 65604, 65373, 321, 32783, 35952, 2221, 59872] avg=32768.0000 max=65604 max_rank=1 min=18 min_rank=0 imbalance_ratio=2.002075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] avg=32768.0000 max=33097 max_rank=7 min=31852 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32507 max_load_reduction_pct=0.495503 moved_to_replicas=94417 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[72100, 13, 120463, 2, 33102, 32815, 2133, 1516] avg=32768.0000 max=120463 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.676239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31088 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=87366 max_load_reduction_pct=0.725252 moved_to_replicas=126377 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31088 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=87366 max_load_reduction_pct=0.725252 moved_to_replicas=126377 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[72100, 13, 120463, 2, 33102, 32815, 2133, 1516] avg=32768.0000 max=120463 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.676239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31088 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=87366 max_load_reduction_pct=0.725252 moved_to_replicas=126377 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] avg=32768.0000 max=33097 max_rank=7 min=31852 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32507 max_load_reduction_pct=0.495503 moved_to_replicas=94417 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[90518, 24876, 32021, 15849, 32850, 112, 79, 65839] avg=32768.0000 max=90518 max_rank=0 min=79 min_rank=6 imbalance_ratio=2.762390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31088 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=87366 max_load_reduction_pct=0.725252 moved_to_replicas=126377 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[72100, 13, 120463, 2, 33102, 32815, 2133, 1516] avg=32768.0000 max=120463 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.676239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31088 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=87366 max_load_reduction_pct=0.725252 moved_to_replicas=126377 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65524, 65535, 26370, 31963, 8, 10646, 29199, 32899] avg=32768.0000 max=65535 max_rank=1 min=8 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33096, 32049, 32021, 33096, 32850, 33095, 32841, 33096] avg=32768.0000 max=33096 max_rank=0 min=32021 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57422 max_load_reduction_pct=0.634371 moved_to_replicas=90165 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65524, 65535, 26370, 31963, 8, 10646, 29199, 32899] avg=32768.0000 max=65535 max_rank=1 min=8 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65524, 65535, 26370, 31963, 8, 10646, 29199, 32899] avg=32768.0000 max=65535 max_rank=1 min=8 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[72100, 13, 120463, 2, 33102, 32815, 2133, 1516] avg=32768.0000 max=120463 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.676239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31088 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=87366 max_load_reduction_pct=0.725252 moved_to_replicas=126377 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65524, 65535, 26370, 31963, 8, 10646, 29199, 32899] avg=32768.0000 max=65535 max_rank=1 min=8 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] avg=32768.0000 max=33097 max_rank=2 min=31963 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64867 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] avg=32768.0000 max=33097 max_rank=2 min=31963 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64867 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] avg=32768.0000 max=33097 max_rank=2 min=31963 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64867 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65524, 65535, 26370, 31963, 8, 10646, 29199, 32899] avg=32768.0000 max=65535 max_rank=1 min=8 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31088 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=87366 max_load_reduction_pct=0.725252 moved_to_replicas=126377 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] avg=32768.0000 max=33097 max_rank=2 min=31963 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64867 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[18, 65604, 65373, 321, 32783, 35952, 2221, 59872] avg=32768.0000 max=65604 max_rank=1 min=18 min_rank=0 imbalance_ratio=2.002075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65524, 65535, 26370, 31963, 8, 10646, 29199, 32899] avg=32768.0000 max=65535 max_rank=1 min=8 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] avg=32768.0000 max=33097 max_rank=2 min=31963 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64867 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[0, 32731, 65525, 25777, 10, 36628, 27987, 73486] avg=32768.0000 max=73486 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.242615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[0, 32731, 65525, 25777, 10, 36628, 27987, 73486] avg=32768.0000 max=73486 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.242615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[0, 32731, 65525, 25777, 10, 36628, 27987, 73486] avg=32768.0000 max=73486 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.242615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32526, 33096, 33096, 32598, 32783, 33096, 31852, 33097] avg=32768.0000 max=33097 max_rank=7 min=31852 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32507 max_load_reduction_pct=0.495503 moved_to_replicas=94417 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65524, 65535, 26370, 31963, 8, 10646, 29199, 32899] avg=32768.0000 max=65535 max_rank=1 min=8 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] avg=32768.0000 max=33097 max_rank=2 min=31963 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64867 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[0, 32731, 65525, 25777, 10, 36628, 27987, 73486] avg=32768.0000 max=73486 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.242615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] avg=32768.0000 max=33096 max_rank=2 min=31519 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40390 max_load_reduction_pct=0.549629 moved_to_replicas=76351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] avg=32768.0000 max=33096 max_rank=2 min=31519 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40390 max_load_reduction_pct=0.549629 moved_to_replicas=76351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] avg=32768.0000 max=33096 max_rank=2 min=31519 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40390 max_load_reduction_pct=0.549629 moved_to_replicas=76351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[0, 32731, 65525, 25777, 10, 36628, 27987, 73486] avg=32768.0000 max=73486 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.242615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] avg=32768.0000 max=33097 max_rank=2 min=31963 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64867 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] avg=32768.0000 max=33096 max_rank=2 min=31519 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40390 max_load_reduction_pct=0.549629 moved_to_replicas=76351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[72100, 13, 120463, 2, 33102, 32815, 2133, 1516] avg=32768.0000 max=120463 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.676239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[0, 32731, 65525, 25777, 10, 36628, 27987, 73486] avg=32768.0000 max=73486 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.242615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] avg=32768.0000 max=33096 max_rank=2 min=31519 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40390 max_load_reduction_pct=0.549629 moved_to_replicas=76351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[12, 13129, 98009, 65444, 32776, 32770, 19986, 18] avg=32768.0000 max=98009 max_rank=2 min=12 min_rank=0 imbalance_ratio=2.990997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[12, 13129, 98009, 65444, 32776, 32770, 19986, 18] avg=32768.0000 max=98009 max_rank=2 min=12 min_rank=0 imbalance_ratio=2.990997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[12, 13129, 98009, 65444, 32776, 32770, 19986, 18] avg=32768.0000 max=98009 max_rank=2 min=12 min_rank=0 imbalance_ratio=2.990997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[33096, 31088, 33097, 32761, 33096, 32815, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31088 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=87366 max_load_reduction_pct=0.725252 moved_to_replicas=126377 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] avg=32768.0000 max=33096 max_rank=2 min=31519 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40390 max_load_reduction_pct=0.549629 moved_to_replicas=76351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[0, 32731, 65525, 25777, 10, 36628, 27987, 73486] avg=32768.0000 max=73486 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.242615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[12, 13129, 98009, 65444, 32776, 32770, 19986, 18] avg=32768.0000 max=98009 max_rank=2 min=12 min_rank=0 imbalance_ratio=2.990997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64913 max_load_reduction_pct=0.662317 moved_to_replicas=97261 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64913 max_load_reduction_pct=0.662317 moved_to_replicas=97261 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64913 max_load_reduction_pct=0.662317 moved_to_replicas=97261 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[12, 13129, 98009, 65444, 32776, 32770, 19986, 18] avg=32768.0000 max=98009 max_rank=2 min=12 min_rank=0 imbalance_ratio=2.990997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] avg=32768.0000 max=33096 max_rank=2 min=31519 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40390 max_load_reduction_pct=0.549629 moved_to_replicas=76351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64913 max_load_reduction_pct=0.662317 moved_to_replicas=97261 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65524, 65535, 26370, 31963, 8, 10646, 29199, 32899] avg=32768.0000 max=65535 max_rank=1 min=8 min_rank=4 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[12, 13129, 98009, 65444, 32776, 32770, 19986, 18] avg=32768.0000 max=98009 max_rank=2 min=12 min_rank=0 imbalance_ratio=2.990997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[33244, 32747, 32769, 32721, 32765, 31175, 55495, 11228] avg=32768.0000 max=55495 max_rank=6 min=11228 min_rank=7 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64913 max_load_reduction_pct=0.662317 moved_to_replicas=97261 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[33244, 32747, 32769, 32721, 32765, 31175, 55495, 11228] avg=32768.0000 max=55495 max_rank=6 min=11228 min_rank=7 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[33244, 32747, 32769, 32721, 32765, 31175, 55495, 11228] avg=32768.0000 max=55495 max_rank=6 min=11228 min_rank=7 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64913 max_load_reduction_pct=0.662317 moved_to_replicas=97261 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33096, 33096, 33097, 31963, 32447, 33095, 32451, 32899] avg=32768.0000 max=33097 max_rank=2 min=31963 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32438 max_load_reduction_pct=0.494972 moved_to_replicas=64867 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[12, 13129, 98009, 65444, 32776, 32770, 19986, 18] avg=32768.0000 max=98009 max_rank=2 min=12 min_rank=0 imbalance_ratio=2.990997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[33244, 32747, 32769, 32721, 32765, 31175, 55495, 11228] avg=32768.0000 max=55495 max_rank=6 min=11228 min_rank=7 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31854 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22399 max_load_reduction_pct=0.403622 moved_to_replicas=22547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31854 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22399 max_load_reduction_pct=0.403622 moved_to_replicas=22547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31854 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22399 max_load_reduction_pct=0.403622 moved_to_replicas=22547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[33244, 32747, 32769, 32721, 32765, 31175, 55495, 11228] avg=32768.0000 max=55495 max_rank=6 min=11228 min_rank=7 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64913 max_load_reduction_pct=0.662317 moved_to_replicas=97261 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31854 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22399 max_load_reduction_pct=0.403622 moved_to_replicas=22547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[33244, 32747, 32769, 32721, 32765, 31175, 55495, 11228] avg=32768.0000 max=55495 max_rank=6 min=11228 min_rank=7 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[0, 32731, 65525, 25777, 10, 36628, 27987, 73486] avg=32768.0000 max=73486 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.242615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31854 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22399 max_load_reduction_pct=0.403622 moved_to_replicas=22547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 79327, 32763, 8, 52211, 16, 32768] avg=32768.0000 max=79327 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.420868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 79327, 32763, 8, 52211, 16, 32768] avg=32768.0000 max=79327 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.420868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 79327, 32763, 8, 52211, 16, 32768] avg=32768.0000 max=79327 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.420868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31854 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22399 max_load_reduction_pct=0.403622 moved_to_replicas=22547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[33244, 32747, 32769, 32721, 32765, 31175, 55495, 11228] avg=32768.0000 max=55495 max_rank=6 min=11228 min_rank=7 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[32761, 32731, 33096, 32749, 33096, 33096, 31519, 33096] avg=32768.0000 max=33096 max_rank=2 min=31519 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=40390 max_load_reduction_pct=0.549629 moved_to_replicas=76351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 79327, 32763, 8, 52211, 16, 32768] avg=32768.0000 max=79327 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.420868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] avg=32768.0000 max=33098 max_rank=2 min=32283 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=46229 max_load_reduction_pct=0.582765 moved_to_replicas=65346 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] avg=32768.0000 max=33098 max_rank=2 min=32283 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=46229 max_load_reduction_pct=0.582765 moved_to_replicas=65346 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] avg=32768.0000 max=33098 max_rank=2 min=32283 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=46229 max_load_reduction_pct=0.582765 moved_to_replicas=65346 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 79327, 32763, 8, 52211, 16, 32768] avg=32768.0000 max=79327 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.420868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31854 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22399 max_load_reduction_pct=0.403622 moved_to_replicas=22547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] avg=32768.0000 max=33098 max_rank=2 min=32283 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=46229 max_load_reduction_pct=0.582765 moved_to_replicas=65346 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 79327, 32763, 8, 52211, 16, 32768] avg=32768.0000 max=79327 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.420868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] avg=32768.0000 max=33098 max_rank=2 min=32283 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=46229 max_load_reduction_pct=0.582765 moved_to_replicas=65346 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[12, 13129, 98009, 65444, 32776, 32770, 19986, 18] avg=32768.0000 max=98009 max_rank=2 min=12 min_rank=0 imbalance_ratio=2.990997 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[65524, 32729, 32758, 32780, 32766, 32703, 16, 32868] avg=32768.0000 max=65524 max_rank=0 min=16 min_rank=6 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[65524, 32729, 32758, 32780, 32766, 32703, 16, 32868] avg=32768.0000 max=65524 max_rank=0 min=16 min_rank=6 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[65524, 32729, 32758, 32780, 32766, 32703, 16, 32868] avg=32768.0000 max=65524 max_rank=0 min=16 min_rank=6 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] avg=32768.0000 max=33098 max_rank=2 min=32283 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=46229 max_load_reduction_pct=0.582765 moved_to_replicas=65346 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 79327, 32763, 8, 52211, 16, 32768] avg=32768.0000 max=79327 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.420868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[65524, 32729, 32758, 32780, 32766, 32703, 16, 32868] avg=32768.0000 max=65524 max_rank=0 min=16 min_rank=6 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[32774, 33096, 33096, 33096, 32776, 32770, 32367, 32169] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64913 max_load_reduction_pct=0.662317 moved_to_replicas=97261 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=32428 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=32428 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=32428 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[65524, 32729, 32758, 32780, 32766, 32703, 16, 32868] avg=32768.0000 max=65524 max_rank=0 min=16 min_rank=6 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] avg=32768.0000 max=33098 max_rank=2 min=32283 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=46229 max_load_reduction_pct=0.582765 moved_to_replicas=65346 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=32428 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[65524, 32729, 32758, 32780, 32766, 32703, 16, 32868] avg=32768.0000 max=65524 max_rank=0 min=16 min_rank=6 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[33785, 98343, 32676, 11, 11, 64565, 32724, 29] avg=32768.0000 max=98343 max_rank=1 min=11 min_rank=3 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[33244, 32747, 32769, 32721, 32765, 31175, 55495, 11228] avg=32768.0000 max=55495 max_rank=6 min=11228 min_rank=7 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=32428 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[33785, 98343, 32676, 11, 11, 64565, 32724, 29] avg=32768.0000 max=98343 max_rank=1 min=11 min_rank=3 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[33785, 98343, 32676, 11, 11, 64565, 32724, 29] avg=32768.0000 max=98343 max_rank=1 min=11 min_rank=3 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=32428 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[65524, 32729, 32758, 32780, 32766, 32703, 16, 32868] avg=32768.0000 max=65524 max_rank=0 min=16 min_rank=6 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[33785, 98343, 32676, 11, 11, 64565, 32724, 29] avg=32768.0000 max=98343 max_rank=1 min=11 min_rank=3 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] avg=32768.0000 max=33096 max_rank=0 min=32187 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65247 max_load_reduction_pct=0.663464 moved_to_replicas=97405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[33096, 32747, 32769, 32721, 32765, 31854, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31854 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22399 max_load_reduction_pct=0.403622 moved_to_replicas=22547 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] avg=32768.0000 max=33096 max_rank=0 min=32187 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65247 max_load_reduction_pct=0.663464 moved_to_replicas=97405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] avg=32768.0000 max=33096 max_rank=0 min=32187 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65247 max_load_reduction_pct=0.663464 moved_to_replicas=97405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=32428 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[33785, 98343, 32676, 11, 11, 64565, 32724, 29] avg=32768.0000 max=98343 max_rank=1 min=11 min_rank=3 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[33785, 98343, 32676, 11, 11, 64565, 32724, 29] avg=32768.0000 max=98343 max_rank=1 min=11 min_rank=3 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] avg=32768.0000 max=33096 max_rank=0 min=32187 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65247 max_load_reduction_pct=0.663464 moved_to_replicas=97405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[33848, 0, 65480, 64638, 27562, 46261, 1193, 23162] avg=32768.0000 max=65480 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] avg=32768.0000 max=33096 max_rank=0 min=32187 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65247 max_load_reduction_pct=0.663464 moved_to_replicas=97405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] avg=32768.0000 max=33096 max_rank=0 min=32187 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65247 max_load_reduction_pct=0.663464 moved_to_replicas=97405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[33848, 0, 65480, 64638, 27562, 46261, 1193, 23162] avg=32768.0000 max=65480 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 79327, 32763, 8, 52211, 16, 32768] avg=32768.0000 max=79327 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.420868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[33848, 0, 65480, 64638, 27562, 46261, 1193, 23162] avg=32768.0000 max=65480 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[33785, 98343, 32676, 11, 11, 64565, 32724, 29] avg=32768.0000 max=98343 max_rank=1 min=11 min_rank=3 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[33848, 0, 65480, 64638, 27562, 46261, 1193, 23162] avg=32768.0000 max=65480 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31545 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32383 max_load_reduction_pct=0.494548 moved_to_replicas=77843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31545 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32383 max_load_reduction_pct=0.494548 moved_to_replicas=77843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32768, 32283, 33098, 32763, 32764, 33095, 32605, 32768] avg=32768.0000 max=33098 max_rank=2 min=32283 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=46229 max_load_reduction_pct=0.582765 moved_to_replicas=65346 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31545 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32383 max_load_reduction_pct=0.494548 moved_to_replicas=77843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] avg=32768.0000 max=33096 max_rank=0 min=32187 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65247 max_load_reduction_pct=0.663464 moved_to_replicas=97405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[33848, 0, 65480, 64638, 27562, 46261, 1193, 23162] avg=32768.0000 max=65480 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[33848, 0, 65480, 64638, 27562, 46261, 1193, 23162] avg=32768.0000 max=65480 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31545 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32383 max_load_reduction_pct=0.494548 moved_to_replicas=77843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[32793, 15443, 11, 64325, 18051, 33451, 65303, 32767] avg=32768.0000 max=65303 max_rank=6 min=11 min_rank=2 imbalance_ratio=1.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31545 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32383 max_load_reduction_pct=0.494548 moved_to_replicas=77843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[32793, 15443, 11, 64325, 18051, 33451, 65303, 32767] avg=32768.0000 max=65303 max_rank=6 min=11 min_rank=2 imbalance_ratio=1.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31545 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32383 max_load_reduction_pct=0.494548 moved_to_replicas=77843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[32793, 15443, 11, 64325, 18051, 33451, 65303, 32767] avg=32768.0000 max=65303 max_rank=6 min=11 min_rank=2 imbalance_ratio=1.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[65524, 32729, 32758, 32780, 32766, 32703, 16, 32868] avg=32768.0000 max=65524 max_rank=0 min=16 min_rank=6 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[33848, 0, 65480, 64638, 27562, 46261, 1193, 23162] avg=32768.0000 max=65480 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] avg=32768.0000 max=33096 max_rank=3 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32207 max_load_reduction_pct=0.493193 moved_to_replicas=63791 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[32793, 15443, 11, 64325, 18051, 33451, 65303, 32767] avg=32768.0000 max=65303 max_rank=6 min=11 min_rank=2 imbalance_ratio=1.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] avg=32768.0000 max=33096 max_rank=3 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32207 max_load_reduction_pct=0.493193 moved_to_replicas=63791 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] avg=32768.0000 max=33096 max_rank=3 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32207 max_load_reduction_pct=0.493193 moved_to_replicas=63791 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[33096, 32729, 32758, 32780, 32766, 32703, 32444, 32868] avg=32768.0000 max=33096 max_rank=0 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32428 max_load_reduction_pct=0.494903 moved_to_replicas=32428 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31545 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32383 max_load_reduction_pct=0.494548 moved_to_replicas=77843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[32793, 15443, 11, 64325, 18051, 33451, 65303, 32767] avg=32768.0000 max=65303 max_rank=6 min=11 min_rank=2 imbalance_ratio=1.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] avg=32768.0000 max=33096 max_rank=3 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32207 max_load_reduction_pct=0.493193 moved_to_replicas=63791 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[32793, 15443, 11, 64325, 18051, 33451, 65303, 32767] avg=32768.0000 max=65303 max_rank=6 min=11 min_rank=2 imbalance_ratio=1.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[52272, 29006, 27, 65536, 10, 82525, 32763, 5] avg=32768.0000 max=82525 max_rank=5 min=5 min_rank=7 imbalance_ratio=2.518463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[52272, 29006, 27, 65536, 10, 82525, 32763, 5] avg=32768.0000 max=82525 max_rank=5 min=5 min_rank=7 imbalance_ratio=2.518463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] avg=32768.0000 max=33096 max_rank=3 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32207 max_load_reduction_pct=0.493193 moved_to_replicas=63791 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[52272, 29006, 27, 65536, 10, 82525, 32763, 5] avg=32768.0000 max=82525 max_rank=5 min=5 min_rank=7 imbalance_ratio=2.518463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] avg=32768.0000 max=33096 max_rank=3 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32207 max_load_reduction_pct=0.493193 moved_to_replicas=63791 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[33785, 98343, 32676, 11, 11, 64565, 32724, 29] avg=32768.0000 max=98343 max_rank=1 min=11 min_rank=3 imbalance_ratio=3.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[32793, 15443, 11, 64325, 18051, 33451, 65303, 32767] avg=32768.0000 max=65303 max_rank=6 min=11 min_rank=2 imbalance_ratio=1.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] avg=32768.0000 max=33097 max_rank=0 min=31763 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49428 max_load_reduction_pct=0.598946 moved_to_replicas=101045 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[52272, 29006, 27, 65536, 10, 82525, 32763, 5] avg=32768.0000 max=82525 max_rank=5 min=5 min_rank=7 imbalance_ratio=2.518463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] avg=32768.0000 max=33097 max_rank=0 min=31763 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49428 max_load_reduction_pct=0.598946 moved_to_replicas=101045 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] avg=32768.0000 max=33097 max_rank=0 min=31763 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49428 max_load_reduction_pct=0.598946 moved_to_replicas=101045 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[33096, 33096, 32676, 32773, 32496, 33096, 32724, 32187] avg=32768.0000 max=33096 max_rank=0 min=32187 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65247 max_load_reduction_pct=0.663464 moved_to_replicas=97405 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] avg=32768.0000 max=33096 max_rank=3 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32207 max_load_reduction_pct=0.493193 moved_to_replicas=63791 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] avg=32768.0000 max=33097 max_rank=0 min=31763 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49428 max_load_reduction_pct=0.598946 moved_to_replicas=101045 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[52272, 29006, 27, 65536, 10, 82525, 32763, 5] avg=32768.0000 max=82525 max_rank=5 min=5 min_rank=7 imbalance_ratio=2.518463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[52272, 29006, 27, 65536, 10, 82525, 32763, 5] avg=32768.0000 max=82525 max_rank=5 min=5 min_rank=7 imbalance_ratio=2.518463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[18319, 31393, 21, 12, 32767, 32763, 32777, 114092] avg=32768.0000 max=114092 max_rank=7 min=12 min_rank=3 imbalance_ratio=3.481812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[18319, 31393, 21, 12, 32767, 32763, 32777, 114092] avg=32768.0000 max=114092 max_rank=7 min=12 min_rank=3 imbalance_ratio=3.481812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[18319, 31393, 21, 12, 32767, 32763, 32777, 114092] avg=32768.0000 max=114092 max_rank=7 min=12 min_rank=3 imbalance_ratio=3.481812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] avg=32768.0000 max=33097 max_rank=0 min=31763 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49428 max_load_reduction_pct=0.598946 moved_to_replicas=101045 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] avg=32768.0000 max=33097 max_rank=0 min=31763 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49428 max_load_reduction_pct=0.598946 moved_to_replicas=101045 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] avg=32768.0000 max=33097 max_rank=7 min=32089 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=80995 max_load_reduction_pct=0.709910 moved_to_replicas=80996 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[52272, 29006, 27, 65536, 10, 82525, 32763, 5] avg=32768.0000 max=82525 max_rank=5 min=5 min_rank=7 imbalance_ratio=2.518463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[33848, 0, 65480, 64638, 27562, 46261, 1193, 23162] avg=32768.0000 max=65480 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[18319, 31393, 21, 12, 32767, 32763, 32777, 114092] avg=32768.0000 max=114092 max_rank=7 min=12 min_rank=3 imbalance_ratio=3.481812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] avg=32768.0000 max=33097 max_rank=7 min=32089 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=80995 max_load_reduction_pct=0.709910 moved_to_replicas=80996 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] avg=32768.0000 max=33097 max_rank=7 min=32089 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=80995 max_load_reduction_pct=0.709910 moved_to_replicas=80996 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] avg=32768.0000 max=33097 max_rank=0 min=31763 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49428 max_load_reduction_pct=0.598946 moved_to_replicas=101045 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32384, 33096, 33096, 31545, 33095, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31545 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32383 max_load_reduction_pct=0.494548 moved_to_replicas=77843 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] avg=32768.0000 max=33097 max_rank=7 min=32089 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=80995 max_load_reduction_pct=0.709910 moved_to_replicas=80996 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[18319, 31393, 21, 12, 32767, 32763, 32777, 114092] avg=32768.0000 max=114092 max_rank=7 min=12 min_rank=3 imbalance_ratio=3.481812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[18319, 31393, 21, 12, 32767, 32763, 32777, 114092] avg=32768.0000 max=114092 max_rank=7 min=12 min_rank=3 imbalance_ratio=3.481812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[10, 32773, 26, 39450, 32758, 1, 32767, 124359] avg=32768.0000 max=124359 max_rank=7 min=1 min_rank=5 imbalance_ratio=3.795135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[10, 32773, 26, 39450, 32758, 1, 32767, 124359] avg=32768.0000 max=124359 max_rank=7 min=1 min_rank=5 imbalance_ratio=3.795135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[10, 32773, 26, 39450, 32758, 1, 32767, 124359] avg=32768.0000 max=124359 max_rank=7 min=1 min_rank=5 imbalance_ratio=3.795135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] avg=32768.0000 max=33097 max_rank=7 min=32089 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=80995 max_load_reduction_pct=0.709910 moved_to_replicas=80996 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] avg=32768.0000 max=33097 max_rank=7 min=32089 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=80995 max_load_reduction_pct=0.709910 moved_to_replicas=80996 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] avg=32768.0000 max=33096 max_rank=3 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=91263 max_load_reduction_pct=0.733867 moved_to_replicas=97617 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[18319, 31393, 21, 12, 32767, 32763, 32777, 114092] avg=32768.0000 max=114092 max_rank=7 min=12 min_rank=3 imbalance_ratio=3.481812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[32793, 15443, 11, 64325, 18051, 33451, 65303, 32767] avg=32768.0000 max=65303 max_rank=6 min=11 min_rank=2 imbalance_ratio=1.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[10, 32773, 26, 39450, 32758, 1, 32767, 124359] avg=32768.0000 max=124359 max_rank=7 min=1 min_rank=5 imbalance_ratio=3.795135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] avg=32768.0000 max=33096 max_rank=3 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=91263 max_load_reduction_pct=0.733867 moved_to_replicas=97617 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] avg=32768.0000 max=33096 max_rank=3 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=91263 max_load_reduction_pct=0.733867 moved_to_replicas=97617 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] avg=32768.0000 max=33097 max_rank=7 min=32089 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=80995 max_load_reduction_pct=0.709910 moved_to_replicas=80996 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[32793, 33095, 32218, 33096, 31983, 33096, 33096, 32767] avg=32768.0000 max=33096 max_rank=3 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32207 max_load_reduction_pct=0.493193 moved_to_replicas=63791 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] avg=32768.0000 max=33096 max_rank=3 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=91263 max_load_reduction_pct=0.733867 moved_to_replicas=97617 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[10, 32773, 26, 39450, 32758, 1, 32767, 124359] avg=32768.0000 max=124359 max_rank=7 min=1 min_rank=5 imbalance_ratio=3.795135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[10, 32773, 26, 39450, 32758, 1, 32767, 124359] avg=32768.0000 max=124359 max_rank=7 min=1 min_rank=5 imbalance_ratio=3.795135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[32539, 65543, 1293, 64401, 32760, 0, 32768, 32840] avg=32768.0000 max=65543 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[32539, 65543, 1293, 64401, 32760, 0, 32768, 32840] avg=32768.0000 max=65543 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[32539, 65543, 1293, 64401, 32760, 0, 32768, 32840] avg=32768.0000 max=65543 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] avg=32768.0000 max=33096 max_rank=3 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=91263 max_load_reduction_pct=0.733867 moved_to_replicas=97617 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] avg=32768.0000 max=33096 max_rank=3 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=91263 max_load_reduction_pct=0.733867 moved_to_replicas=97617 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] avg=32768.0000 max=33096 max_rank=1 min=32447 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=63752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[32539, 65543, 1293, 64401, 32760, 0, 32768, 32840] avg=32768.0000 max=65543 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[10, 32773, 26, 39450, 32758, 1, 32767, 124359] avg=32768.0000 max=124359 max_rank=7 min=1 min_rank=5 imbalance_ratio=3.795135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[52272, 29006, 27, 65536, 10, 82525, 32763, 5] avg=32768.0000 max=82525 max_rank=5 min=5 min_rank=7 imbalance_ratio=2.518463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] avg=32768.0000 max=33096 max_rank=1 min=32447 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=63752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] avg=32768.0000 max=33096 max_rank=1 min=32447 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=63752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] avg=32768.0000 max=33096 max_rank=1 min=32447 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=63752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] avg=32768.0000 max=33096 max_rank=3 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=91263 max_load_reduction_pct=0.733867 moved_to_replicas=97617 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[32539, 65543, 1293, 64401, 32760, 0, 32768, 32840] avg=32768.0000 max=65543 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[33097, 31763, 32467, 33096, 33094, 33097, 32763, 32767] avg=32768.0000 max=33097 max_rank=0 min=31763 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49428 max_load_reduction_pct=0.598946 moved_to_replicas=101045 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[32539, 65543, 1293, 64401, 32760, 0, 32768, 32840] avg=32768.0000 max=65543 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[32736, 7, 32764, 65508, 32785, 32789, 65523, 32] avg=32768.0000 max=65523 max_rank=6 min=7 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[32736, 7, 32764, 65508, 32785, 32789, 65523, 32] avg=32768.0000 max=65523 max_rank=6 min=7 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[32736, 7, 32764, 65508, 32785, 32789, 65523, 32] avg=32768.0000 max=65523 max_rank=6 min=7 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] avg=32768.0000 max=33096 max_rank=1 min=32447 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=63752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] avg=32768.0000 max=33096 max_rank=1 min=32447 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=63752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] avg=32768.0000 max=33096 max_rank=3 min=32434 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64839 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[32736, 7, 32764, 65508, 32785, 32789, 65523, 32] avg=32768.0000 max=65523 max_rank=6 min=7 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] avg=32768.0000 max=33096 max_rank=3 min=32434 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64839 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] avg=32768.0000 max=33096 max_rank=3 min=32434 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64839 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[32539, 65543, 1293, 64401, 32760, 0, 32768, 32840] avg=32768.0000 max=65543 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[18319, 31393, 21, 12, 32767, 32763, 32777, 114092] avg=32768.0000 max=114092 max_rank=7 min=12 min_rank=3 imbalance_ratio=3.481812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] avg=32768.0000 max=33096 max_rank=3 min=32434 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64839 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[32736, 7, 32764, 65508, 32785, 32789, 65523, 32] avg=32768.0000 max=65523 max_rank=6 min=7 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] avg=32768.0000 max=33096 max_rank=1 min=32447 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=63752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[33095, 32089, 32782, 32774, 32767, 32763, 32777, 33097] avg=32768.0000 max=33097 max_rank=7 min=32089 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=80995 max_load_reduction_pct=0.709910 moved_to_replicas=80996 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[32736, 7, 32764, 65508, 32785, 32789, 65523, 32] avg=32768.0000 max=65523 max_rank=6 min=7 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[32763, 64612, 33679, 33, 32800, 32711, 32759, 32787] avg=32768.0000 max=64612 max_rank=1 min=33 min_rank=3 imbalance_ratio=1.971802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[32763, 64612, 33679, 33, 32800, 32711, 32759, 32787] avg=32768.0000 max=64612 max_rank=1 min=33 min_rank=3 imbalance_ratio=1.971802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[32763, 64612, 33679, 33, 32800, 32711, 32759, 32787] avg=32768.0000 max=64612 max_rank=1 min=33 min_rank=3 imbalance_ratio=1.971802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] avg=32768.0000 max=33096 max_rank=3 min=32434 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64839 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] avg=32768.0000 max=33096 max_rank=1 min=32132 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31516 max_load_reduction_pct=0.487773 moved_to_replicas=32099 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] avg=32768.0000 max=33096 max_rank=3 min=32434 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64839 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[32763, 64612, 33679, 33, 32800, 32711, 32759, 32787] avg=32768.0000 max=64612 max_rank=1 min=33 min_rank=3 imbalance_ratio=1.971802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] avg=32768.0000 max=33096 max_rank=1 min=32132 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31516 max_load_reduction_pct=0.487773 moved_to_replicas=32099 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] avg=32768.0000 max=33096 max_rank=1 min=32132 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31516 max_load_reduction_pct=0.487773 moved_to_replicas=32099 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[32736, 7, 32764, 65508, 32785, 32789, 65523, 32] avg=32768.0000 max=65523 max_rank=6 min=7 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[10, 32773, 26, 39450, 32758, 1, 32767, 124359] avg=32768.0000 max=124359 max_rank=7 min=1 min_rank=5 imbalance_ratio=3.795135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[32763, 64612, 33679, 33, 32800, 32711, 32759, 32787] avg=32768.0000 max=64612 max_rank=1 min=33 min_rank=3 imbalance_ratio=1.971802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] avg=32768.0000 max=33096 max_rank=1 min=32132 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31516 max_load_reduction_pct=0.487773 moved_to_replicas=32099 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] avg=32768.0000 max=33096 max_rank=3 min=32434 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64839 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[5, 0, 32771, 65513, 6, 65621, 64110, 34118] avg=32768.0000 max=65621 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[32770, 32773, 32122, 33096, 32758, 32763, 32767, 33095] avg=32768.0000 max=33096 max_rank=3 min=32122 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=91263 max_load_reduction_pct=0.733867 moved_to_replicas=97617 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[32763, 64612, 33679, 33, 32800, 32711, 32759, 32787] avg=32768.0000 max=64612 max_rank=1 min=33 min_rank=3 imbalance_ratio=1.971802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[5, 0, 32771, 65513, 6, 65621, 64110, 34118] avg=32768.0000 max=65621 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[5, 0, 32771, 65513, 6, 65621, 64110, 34118] avg=32768.0000 max=65621 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] avg=32768.0000 max=33096 max_rank=1 min=32132 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31516 max_load_reduction_pct=0.487773 moved_to_replicas=32099 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32525 max_load_reduction_pct=0.495649 moved_to_replicas=96978 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[5, 0, 32771, 65513, 6, 65621, 64110, 34118] avg=32768.0000 max=65621 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] avg=32768.0000 max=33096 max_rank=1 min=32132 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31516 max_load_reduction_pct=0.487773 moved_to_replicas=32099 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32525 max_load_reduction_pct=0.495649 moved_to_replicas=96978 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[32763, 64612, 33679, 33, 32800, 32711, 32759, 32787] avg=32768.0000 max=64612 max_rank=1 min=33 min_rank=3 imbalance_ratio=1.971802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32525 max_load_reduction_pct=0.495649 moved_to_replicas=96978 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[32539, 65543, 1293, 64401, 32760, 0, 32768, 32840] avg=32768.0000 max=65543 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[5, 0, 32771, 65513, 6, 65621, 64110, 34118] avg=32768.0000 max=65621 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32525 max_load_reduction_pct=0.495649 moved_to_replicas=96978 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] avg=32768.0000 max=33096 max_rank=1 min=32132 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31516 max_load_reduction_pct=0.487773 moved_to_replicas=32099 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=25 tokens_per_ep_rank=[2, 32558, 69417, 46875, 98288, 5, 14988, 11] avg=32768.0000 max=98288 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=25 tokens_per_ep_rank=[2, 32558, 69417, 46875, 98288, 5, 14988, 11] avg=32768.0000 max=98288 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[32539, 33096, 32598, 33096, 32760, 32447, 32768, 32840] avg=32768.0000 max=33096 max_rank=1 min=32447 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=63752 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[5, 0, 32771, 65513, 6, 65621, 64110, 34118] avg=32768.0000 max=65621 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=25 tokens_per_ep_rank=[2, 32558, 69417, 46875, 98288, 5, 14988, 11] avg=32768.0000 max=98288 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32525 max_load_reduction_pct=0.495649 moved_to_replicas=96978 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=25 tokens_per_ep_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] avg=32768.0000 max=33097 max_rank=3 min=32325 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65191 max_load_reduction_pct=0.663265 moved_to_replicas=115292 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=25 tokens_per_ep_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] avg=32768.0000 max=33097 max_rank=3 min=32325 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65191 max_load_reduction_pct=0.663265 moved_to_replicas=115292 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=25 tokens_per_ep_rank=[2, 32558, 69417, 46875, 98288, 5, 14988, 11] avg=32768.0000 max=98288 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32525 max_load_reduction_pct=0.495649 moved_to_replicas=96978 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=25 tokens_per_ep_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] avg=32768.0000 max=33097 max_rank=3 min=32325 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65191 max_load_reduction_pct=0.663265 moved_to_replicas=115292 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[5, 0, 32771, 65513, 6, 65621, 64110, 34118] avg=32768.0000 max=65621 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[32736, 7, 32764, 65508, 32785, 32789, 65523, 32] avg=32768.0000 max=65523 max_rank=6 min=7 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=25 tokens_per_ep_rank=[2, 32558, 69417, 46875, 98288, 5, 14988, 11] avg=32768.0000 max=98288 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=25 tokens_per_ep_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] avg=32768.0000 max=33097 max_rank=3 min=32325 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65191 max_load_reduction_pct=0.663265 moved_to_replicas=115292 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32525 max_load_reduction_pct=0.495649 moved_to_replicas=96978 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31366, 6, 66934, 65494, 1] avg=32768.0000 max=66934 max_rank=5 min=1 min_rank=7 imbalance_ratio=2.042664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31366, 6, 66934, 65494, 1] avg=32768.0000 max=66934 max_rank=5 min=1 min_rank=7 imbalance_ratio=2.042664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=25 tokens_per_ep_rank=[2, 32558, 69417, 46875, 98288, 5, 14988, 11] avg=32768.0000 max=98288 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[32736, 32434, 32764, 33096, 32785, 32789, 33096, 32444] avg=32768.0000 max=33096 max_rank=3 min=32434 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64839 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31366, 6, 66934, 65494, 1] avg=32768.0000 max=66934 max_rank=5 min=1 min_rank=7 imbalance_ratio=2.042664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=25 tokens_per_ep_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] avg=32768.0000 max=33097 max_rank=3 min=32325 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65191 max_load_reduction_pct=0.663265 moved_to_replicas=115292 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=4 min=31750 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33838 max_load_reduction_pct=0.505543 moved_to_replicas=66236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=4 min=31750 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33838 max_load_reduction_pct=0.505543 moved_to_replicas=66236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=25 tokens_per_ep_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] avg=32768.0000 max=33097 max_rank=3 min=32325 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65191 max_load_reduction_pct=0.663265 moved_to_replicas=115292 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31366, 6, 66934, 65494, 1] avg=32768.0000 max=66934 max_rank=5 min=1 min_rank=7 imbalance_ratio=2.042664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=4 min=31750 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33838 max_load_reduction_pct=0.505543 moved_to_replicas=66236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=25 tokens_per_ep_rank=[2, 32558, 69417, 46875, 98288, 5, 14988, 11] avg=32768.0000 max=98288 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[32763, 64612, 33679, 33, 32800, 32711, 32759, 32787] avg=32768.0000 max=64612 max_rank=1 min=33 min_rank=3 imbalance_ratio=1.971802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31366, 6, 66934, 65494, 1] avg=32768.0000 max=66934 max_rank=5 min=1 min_rank=7 imbalance_ratio=2.042664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=4 min=31750 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33838 max_load_reduction_pct=0.505543 moved_to_replicas=66236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=27 tokens_per_ep_rank=[10, 32849, 65352, 30863, 1719, 32772, 65808, 32771] avg=32768.0000 max=65808 max_rank=6 min=10 min_rank=0 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=25 tokens_per_ep_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] avg=32768.0000 max=33097 max_rank=3 min=32325 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65191 max_load_reduction_pct=0.663265 moved_to_replicas=115292 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=27 tokens_per_ep_rank=[10, 32849, 65352, 30863, 1719, 32772, 65808, 32771] avg=32768.0000 max=65808 max_rank=6 min=10 min_rank=0 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31366, 6, 66934, 65494, 1] avg=32768.0000 max=66934 max_rank=5 min=1 min_rank=7 imbalance_ratio=2.042664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=27 tokens_per_ep_rank=[10, 32849, 65352, 30863, 1719, 32772, 65808, 32771] avg=32768.0000 max=65808 max_rank=6 min=10 min_rank=0 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=4 min=31750 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33838 max_load_reduction_pct=0.505543 moved_to_replicas=66236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[32763, 33096, 33096, 32132, 32800, 32711, 32759, 32787] avg=32768.0000 max=33096 max_rank=1 min=32132 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31516 max_load_reduction_pct=0.487773 moved_to_replicas=32099 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=27 tokens_per_ep_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] avg=32768.0000 max=33096 max_rank=2 min=31742 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32712 max_load_reduction_pct=0.497082 moved_to_replicas=64968 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=27 tokens_per_ep_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] avg=32768.0000 max=33096 max_rank=2 min=31742 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32712 max_load_reduction_pct=0.497082 moved_to_replicas=64968 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=4 min=31750 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33838 max_load_reduction_pct=0.505543 moved_to_replicas=66236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=27 tokens_per_ep_rank=[10, 32849, 65352, 30863, 1719, 32772, 65808, 32771] avg=32768.0000 max=65808 max_rank=6 min=10 min_rank=0 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=27 tokens_per_ep_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] avg=32768.0000 max=33096 max_rank=2 min=31742 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32712 max_load_reduction_pct=0.497082 moved_to_replicas=64968 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31366, 6, 66934, 65494, 1] avg=32768.0000 max=66934 max_rank=5 min=1 min_rank=7 imbalance_ratio=2.042664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=27 tokens_per_ep_rank=[10, 32849, 65352, 30863, 1719, 32772, 65808, 32771] avg=32768.0000 max=65808 max_rank=6 min=10 min_rank=0 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[5, 0, 32771, 65513, 6, 65621, 64110, 34118] avg=32768.0000 max=65621 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=28 tokens_per_ep_rank=[50016, 49931, 34506, 5, 32765, 22, 29386, 65513] avg=32768.0000 max=65513 max_rank=7 min=5 min_rank=3 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=27 tokens_per_ep_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] avg=32768.0000 max=33096 max_rank=2 min=31742 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32712 max_load_reduction_pct=0.497082 moved_to_replicas=64968 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=28 tokens_per_ep_rank=[50016, 49931, 34506, 5, 32765, 22, 29386, 65513] avg=32768.0000 max=65513 max_rank=7 min=5 min_rank=3 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=4 min=31750 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33838 max_load_reduction_pct=0.505543 moved_to_replicas=66236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=27 tokens_per_ep_rank=[10, 32849, 65352, 30863, 1719, 32772, 65808, 32771] avg=32768.0000 max=65808 max_rank=6 min=10 min_rank=0 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=27 tokens_per_ep_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] avg=32768.0000 max=33096 max_rank=2 min=31742 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32712 max_load_reduction_pct=0.497082 moved_to_replicas=64968 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=28 tokens_per_ep_rank=[50016, 49931, 34506, 5, 32765, 22, 29386, 65513] avg=32768.0000 max=65513 max_rank=7 min=5 min_rank=3 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=28 tokens_per_ep_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] avg=32768.0000 max=33097 max_rank=1 min=31477 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32416 max_load_reduction_pct=0.494803 moved_to_replicas=67582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[32422, 32525, 32771, 33096, 32042, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32525 max_load_reduction_pct=0.495649 moved_to_replicas=96978 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=28 tokens_per_ep_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] avg=32768.0000 max=33097 max_rank=1 min=31477 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32416 max_load_reduction_pct=0.494803 moved_to_replicas=67582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=28 tokens_per_ep_rank=[50016, 49931, 34506, 5, 32765, 22, 29386, 65513] avg=32768.0000 max=65513 max_rank=7 min=5 min_rank=3 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=27 tokens_per_ep_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] avg=32768.0000 max=33096 max_rank=2 min=31742 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32712 max_load_reduction_pct=0.497082 moved_to_replicas=64968 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=28 tokens_per_ep_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] avg=32768.0000 max=33097 max_rank=1 min=31477 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32416 max_load_reduction_pct=0.494803 moved_to_replicas=67582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=27 tokens_per_ep_rank=[10, 32849, 65352, 30863, 1719, 32772, 65808, 32771] avg=32768.0000 max=65808 max_rank=6 min=10 min_rank=0 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=28 tokens_per_ep_rank=[50016, 49931, 34506, 5, 32765, 22, 29386, 65513] avg=32768.0000 max=65513 max_rank=7 min=5 min_rank=3 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=29 tokens_per_ep_rank=[10, 69178, 9, 29031, 65504, 0, 32769, 65643] avg=32768.0000 max=69178 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.111145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=28 tokens_per_ep_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] avg=32768.0000 max=33097 max_rank=1 min=31477 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32416 max_load_reduction_pct=0.494803 moved_to_replicas=67582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=25 tokens_per_ep_rank=[2, 32558, 69417, 46875, 98288, 5, 14988, 11] avg=32768.0000 max=98288 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=29 tokens_per_ep_rank=[10, 69178, 9, 29031, 65504, 0, 32769, 65643] avg=32768.0000 max=69178 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.111145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=27 tokens_per_ep_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] avg=32768.0000 max=33096 max_rank=2 min=31742 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32712 max_load_reduction_pct=0.497082 moved_to_replicas=64968 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=28 tokens_per_ep_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] avg=32768.0000 max=33097 max_rank=1 min=31477 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32416 max_load_reduction_pct=0.494803 moved_to_replicas=67582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=28 tokens_per_ep_rank=[50016, 49931, 34506, 5, 32765, 22, 29386, 65513] avg=32768.0000 max=65513 max_rank=7 min=5 min_rank=3 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=29 tokens_per_ep_rank=[10, 69178, 9, 29031, 65504, 0, 32769, 65643] avg=32768.0000 max=69178 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.111145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=29 tokens_per_ep_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=31672 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36082 max_load_reduction_pct=0.521582 moved_to_replicas=101037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=25 tokens_per_ep_rank=[32764, 32558, 33096, 33097, 33096, 32435, 32325, 32773] avg=32768.0000 max=33097 max_rank=3 min=32325 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65191 max_load_reduction_pct=0.663265 moved_to_replicas=115292 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=29 tokens_per_ep_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=31672 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36082 max_load_reduction_pct=0.521582 moved_to_replicas=101037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=29 tokens_per_ep_rank=[10, 69178, 9, 29031, 65504, 0, 32769, 65643] avg=32768.0000 max=69178 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.111145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=29 tokens_per_ep_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=31672 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36082 max_load_reduction_pct=0.521582 moved_to_replicas=101037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=28 tokens_per_ep_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] avg=32768.0000 max=33097 max_rank=1 min=31477 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32416 max_load_reduction_pct=0.494803 moved_to_replicas=67582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=28 tokens_per_ep_rank=[50016, 49931, 34506, 5, 32765, 22, 29386, 65513] avg=32768.0000 max=65513 max_rank=7 min=5 min_rank=3 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=29 tokens_per_ep_rank=[10, 69178, 9, 29031, 65504, 0, 32769, 65643] avg=32768.0000 max=69178 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.111145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=30 tokens_per_ep_rank=[4426, 32002, 32739, 64955, 60949, 32698, 33128, 1247] avg=32768.0000 max=64955 max_rank=3 min=1247 min_rank=7 imbalance_ratio=1.982269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=29 tokens_per_ep_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=31672 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36082 max_load_reduction_pct=0.521582 moved_to_replicas=101037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=30 tokens_per_ep_rank=[4426, 32002, 32739, 64955, 60949, 32698, 33128, 1247] avg=32768.0000 max=64955 max_rank=3 min=1247 min_rank=7 imbalance_ratio=1.982269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31366, 6, 66934, 65494, 1] avg=32768.0000 max=66934 max_rank=5 min=1 min_rank=7 imbalance_ratio=2.042664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=28 tokens_per_ep_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] avg=32768.0000 max=33097 max_rank=1 min=31477 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32416 max_load_reduction_pct=0.494803 moved_to_replicas=67582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=29 tokens_per_ep_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=31672 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36082 max_load_reduction_pct=0.521582 moved_to_replicas=101037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=30 tokens_per_ep_rank=[4426, 32002, 32739, 64955, 60949, 32698, 33128, 1247] avg=32768.0000 max=64955 max_rank=3 min=1247 min_rank=7 imbalance_ratio=1.982269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=30 tokens_per_ep_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32034 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31859 max_load_reduction_pct=0.490478 moved_to_replicas=59744 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=29 tokens_per_ep_rank=[10, 69178, 9, 29031, 65504, 0, 32769, 65643] avg=32768.0000 max=69178 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.111145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=30 tokens_per_ep_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32034 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31859 max_load_reduction_pct=0.490478 moved_to_replicas=59744 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=26 tokens_per_ep_rank=[32809, 32768, 32766, 31750, 33096, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=4 min=31750 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33838 max_load_reduction_pct=0.505543 moved_to_replicas=66236 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=30 tokens_per_ep_rank=[4426, 32002, 32739, 64955, 60949, 32698, 33128, 1247] avg=32768.0000 max=64955 max_rank=3 min=1247 min_rank=7 imbalance_ratio=1.982269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=30 tokens_per_ep_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32034 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31859 max_load_reduction_pct=0.490478 moved_to_replicas=59744 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=29 tokens_per_ep_rank=[10, 69178, 9, 29031, 65504, 0, 32769, 65643] avg=32768.0000 max=69178 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.111145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=29 tokens_per_ep_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=31672 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36082 max_load_reduction_pct=0.521582 moved_to_replicas=101037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=30 tokens_per_ep_rank=[4426, 32002, 32739, 64955, 60949, 32698, 33128, 1247] avg=32768.0000 max=64955 max_rank=3 min=1247 min_rank=7 imbalance_ratio=1.982269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=31 tokens_per_ep_rank=[15, 98294, 10, 6, 16, 32764, 32755, 98284] avg=32768.0000 max=98294 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=30 tokens_per_ep_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32034 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31859 max_load_reduction_pct=0.490478 moved_to_replicas=59744 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=31 tokens_per_ep_rank=[15, 98294, 10, 6, 16, 32764, 32755, 98284] avg=32768.0000 max=98294 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=27 tokens_per_ep_rank=[10, 32849, 65352, 30863, 1719, 32772, 65808, 32771] avg=32768.0000 max=65808 max_rank=6 min=10 min_rank=0 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=29 tokens_per_ep_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=31672 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36082 max_load_reduction_pct=0.521582 moved_to_replicas=101037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=30 tokens_per_ep_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32034 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31859 max_load_reduction_pct=0.490478 moved_to_replicas=59744 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=31 tokens_per_ep_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=130386 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=31 tokens_per_ep_rank=[15, 98294, 10, 6, 16, 32764, 32755, 98284] avg=32768.0000 max=98294 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=30 tokens_per_ep_rank=[4426, 32002, 32739, 64955, 60949, 32698, 33128, 1247] avg=32768.0000 max=64955 max_rank=3 min=1247 min_rank=7 imbalance_ratio=1.982269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=31 tokens_per_ep_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=130386 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=27 tokens_per_ep_rank=[32722, 32849, 33096, 31742, 33096, 32772, 33096, 32771] avg=32768.0000 max=33096 max_rank=2 min=31742 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32712 max_load_reduction_pct=0.497082 moved_to_replicas=64968 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=31 tokens_per_ep_rank=[15, 98294, 10, 6, 16, 32764, 32755, 98284] avg=32768.0000 max=98294 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=31 tokens_per_ep_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=130386 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=31 tokens_per_ep_rank=[15, 98294, 10, 6, 16, 32764, 32755, 98284] avg=32768.0000 max=98294 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=30 tokens_per_ep_rank=[4426, 32002, 32739, 64955, 60949, 32698, 33128, 1247] avg=32768.0000 max=64955 max_rank=3 min=1247 min_rank=7 imbalance_ratio=1.982269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=30 tokens_per_ep_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32034 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31859 max_load_reduction_pct=0.490478 moved_to_replicas=59744 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 65534, 32769, 32769, 65961, 5, 0] avg=32768.0000 max=65961 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.012970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=31 tokens_per_ep_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=130386 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 65534, 32769, 32769, 65961, 5, 0] avg=32768.0000 max=65961 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.012970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=31 tokens_per_ep_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=130386 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=28 tokens_per_ep_rank=[50016, 49931, 34506, 5, 32765, 22, 29386, 65513] avg=32768.0000 max=65513 max_rank=7 min=5 min_rank=3 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=30 tokens_per_ep_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32034 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31859 max_load_reduction_pct=0.490478 moved_to_replicas=59744 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] avg=32768.0000 max=33096 max_rank=2 min=32376 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32865 max_load_reduction_pct=0.498249 moved_to_replicas=65303 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 65534, 32769, 32769, 65961, 5, 0] avg=32768.0000 max=65961 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.012970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] avg=32768.0000 max=33096 max_rank=2 min=32376 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32865 max_load_reduction_pct=0.498249 moved_to_replicas=65303 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=31 tokens_per_ep_rank=[15, 98294, 10, 6, 16, 32764, 32755, 98284] avg=32768.0000 max=98294 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 65534, 32769, 32769, 65961, 5, 0] avg=32768.0000 max=65961 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.012970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=28 tokens_per_ep_rank=[33094, 33097, 33096, 32422, 32765, 33097, 31477, 33096] avg=32768.0000 max=33097 max_rank=1 min=31477 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32416 max_load_reduction_pct=0.494803 moved_to_replicas=67582 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] avg=32768.0000 max=33096 max_rank=2 min=32376 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32865 max_load_reduction_pct=0.498249 moved_to_replicas=65303 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 65534, 32769, 32769, 65961, 5, 0] avg=32768.0000 max=65961 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.012970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=31 tokens_per_ep_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=130386 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=31 tokens_per_ep_rank=[15, 98294, 10, 6, 16, 32764, 32755, 98284] avg=32768.0000 max=98294 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=33 tokens_per_ep_rank=[62244, 40, 65526, 13, 47, 3321, 98187, 32766] avg=32768.0000 max=98187 max_rank=6 min=13 min_rank=3 imbalance_ratio=2.996429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] avg=32768.0000 max=33096 max_rank=2 min=32376 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32865 max_load_reduction_pct=0.498249 moved_to_replicas=65303 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=33 tokens_per_ep_rank=[62244, 40, 65526, 13, 47, 3321, 98187, 32766] avg=32768.0000 max=98187 max_rank=6 min=13 min_rank=3 imbalance_ratio=2.996429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] avg=32768.0000 max=33096 max_rank=2 min=32376 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32865 max_load_reduction_pct=0.498249 moved_to_replicas=65303 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=29 tokens_per_ep_rank=[10, 69178, 9, 29031, 65504, 0, 32769, 65643] avg=32768.0000 max=69178 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.111145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=33 tokens_per_ep_rank=[62244, 40, 65526, 13, 47, 3321, 98187, 32766] avg=32768.0000 max=98187 max_rank=6 min=13 min_rank=3 imbalance_ratio=2.996429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=31 tokens_per_ep_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=130386 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=33 tokens_per_ep_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] avg=32768.0000 max=33096 max_rank=0 min=32369 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65091 max_load_reduction_pct=0.662929 moved_to_replicas=126669 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=33 tokens_per_ep_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] avg=32768.0000 max=33096 max_rank=0 min=32369 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65091 max_load_reduction_pct=0.662929 moved_to_replicas=126669 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 65534, 32769, 32769, 65961, 5, 0] avg=32768.0000 max=65961 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.012970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=33 tokens_per_ep_rank=[62244, 40, 65526, 13, 47, 3321, 98187, 32766] avg=32768.0000 max=98187 max_rank=6 min=13 min_rank=3 imbalance_ratio=2.996429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=29 tokens_per_ep_rank=[32557, 33096, 33096, 31672, 33096, 32762, 32769, 33096] avg=32768.0000 max=33096 max_rank=1 min=31672 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36082 max_load_reduction_pct=0.521582 moved_to_replicas=101037 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=33 tokens_per_ep_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] avg=32768.0000 max=33096 max_rank=0 min=32369 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65091 max_load_reduction_pct=0.662929 moved_to_replicas=126669 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=33 tokens_per_ep_rank=[62244, 40, 65526, 13, 47, 3321, 98187, 32766] avg=32768.0000 max=98187 max_rank=6 min=13 min_rank=3 imbalance_ratio=2.996429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] avg=32768.0000 max=33096 max_rank=2 min=32376 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32865 max_load_reduction_pct=0.498249 moved_to_replicas=65303 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=34 tokens_per_ep_rank=[65879, 65534, 32419, 5, 21, 0, 0, 98286] avg=32768.0000 max=98286 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=34 tokens_per_ep_rank=[65879, 65534, 32419, 5, 21, 0, 0, 98286] avg=32768.0000 max=98286 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=33 tokens_per_ep_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] avg=32768.0000 max=33096 max_rank=0 min=32369 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65091 max_load_reduction_pct=0.662929 moved_to_replicas=126669 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 65534, 32769, 32769, 65961, 5, 0] avg=32768.0000 max=65961 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.012970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=33 tokens_per_ep_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] avg=32768.0000 max=33096 max_rank=0 min=32369 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65091 max_load_reduction_pct=0.662929 moved_to_replicas=126669 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=30 tokens_per_ep_rank=[4426, 32002, 32739, 64955, 60949, 32698, 33128, 1247] avg=32768.0000 max=64955 max_rank=3 min=1247 min_rank=7 imbalance_ratio=1.982269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=34 tokens_per_ep_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65190 max_load_reduction_pct=0.663268 moved_to_replicas=130411 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=34 tokens_per_ep_rank=[65879, 65534, 32419, 5, 21, 0, 0, 98286] avg=32768.0000 max=98286 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=34 tokens_per_ep_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65190 max_load_reduction_pct=0.663268 moved_to_replicas=130411 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] avg=32768.0000 max=33096 max_rank=2 min=32376 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32865 max_load_reduction_pct=0.498249 moved_to_replicas=65303 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=33 tokens_per_ep_rank=[62244, 40, 65526, 13, 47, 3321, 98187, 32766] avg=32768.0000 max=98187 max_rank=6 min=13 min_rank=3 imbalance_ratio=2.996429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=34 tokens_per_ep_rank=[65879, 65534, 32419, 5, 21, 0, 0, 98286] avg=32768.0000 max=98286 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=30 tokens_per_ep_rank=[32289, 32034, 32739, 33096, 33096, 32698, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32034 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31859 max_load_reduction_pct=0.490478 moved_to_replicas=59744 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=34 tokens_per_ep_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65190 max_load_reduction_pct=0.663268 moved_to_replicas=130411 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=34 tokens_per_ep_rank=[65879, 65534, 32419, 5, 21, 0, 0, 98286] avg=32768.0000 max=98286 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=33 tokens_per_ep_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] avg=32768.0000 max=33096 max_rank=0 min=32369 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65091 max_load_reduction_pct=0.662929 moved_to_replicas=126669 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=35 tokens_per_ep_rank=[10, 31483, 172, 131054, 32867, 11, 66547, 0] avg=32768.0000 max=131054 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=35 tokens_per_ep_rank=[10, 31483, 172, 131054, 32867, 11, 66547, 0] avg=32768.0000 max=131054 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=34 tokens_per_ep_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65190 max_load_reduction_pct=0.663268 moved_to_replicas=130411 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=33 tokens_per_ep_rank=[62244, 40, 65526, 13, 47, 3321, 98187, 32766] avg=32768.0000 max=98187 max_rank=6 min=13 min_rank=3 imbalance_ratio=2.996429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=34 tokens_per_ep_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65190 max_load_reduction_pct=0.663268 moved_to_replicas=130411 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=35 tokens_per_ep_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] avg=32768.0000 max=33096 max_rank=3 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97958 max_load_reduction_pct=0.747463 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=31 tokens_per_ep_rank=[15, 98294, 10, 6, 16, 32764, 32755, 98284] avg=32768.0000 max=98294 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=35 tokens_per_ep_rank=[10, 31483, 172, 131054, 32867, 11, 66547, 0] avg=32768.0000 max=131054 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=35 tokens_per_ep_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] avg=32768.0000 max=33096 max_rank=3 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97958 max_load_reduction_pct=0.747463 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=33 tokens_per_ep_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] avg=32768.0000 max=33096 max_rank=0 min=32369 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65091 max_load_reduction_pct=0.662929 moved_to_replicas=126669 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=34 tokens_per_ep_rank=[65879, 65534, 32419, 5, 21, 0, 0, 98286] avg=32768.0000 max=98286 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=35 tokens_per_ep_rank=[10, 31483, 172, 131054, 32867, 11, 66547, 0] avg=32768.0000 max=131054 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=35 tokens_per_ep_rank=[10, 31483, 172, 131054, 32867, 11, 66547, 0] avg=32768.0000 max=131054 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=31 tokens_per_ep_rank=[32777, 33096, 32446, 32768, 32442, 32764, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65198 max_load_reduction_pct=0.663296 moved_to_replicas=130386 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=35 tokens_per_ep_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] avg=32768.0000 max=33096 max_rank=3 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97958 max_load_reduction_pct=0.747463 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=34 tokens_per_ep_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65190 max_load_reduction_pct=0.663268 moved_to_replicas=130411 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=36 tokens_per_ep_rank=[92882, 42255, 5, 65535, 6, 32776, 5, 28680] avg=32768.0000 max=92882 max_rank=0 min=5 min_rank=2 imbalance_ratio=2.834534 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=35 tokens_per_ep_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] avg=32768.0000 max=33096 max_rank=3 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97958 max_load_reduction_pct=0.747463 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=36 tokens_per_ep_rank=[92882, 42255, 5, 65535, 6, 32776, 5, 28680] avg=32768.0000 max=92882 max_rank=0 min=5 min_rank=2 imbalance_ratio=2.834534 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=34 tokens_per_ep_rank=[65879, 65534, 32419, 5, 21, 0, 0, 98286] avg=32768.0000 max=98286 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=35 tokens_per_ep_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] avg=32768.0000 max=33096 max_rank=3 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97958 max_load_reduction_pct=0.747463 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=36 tokens_per_ep_rank=[92882, 42255, 5, 65535, 6, 32776, 5, 28680] avg=32768.0000 max=92882 max_rank=0 min=5 min_rank=2 imbalance_ratio=2.834534 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=36 tokens_per_ep_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] avg=32768.0000 max=33097 max_rank=6 min=31772 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=59785 max_load_reduction_pct=0.643666 moved_to_replicas=101384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 65534, 32769, 32769, 65961, 5, 0] avg=32768.0000 max=65961 max_rank=5 min=0 min_rank=7 imbalance_ratio=2.012970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=35 tokens_per_ep_rank=[10, 31483, 172, 131054, 32867, 11, 66547, 0] avg=32768.0000 max=131054 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=36 tokens_per_ep_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] avg=32768.0000 max=33097 max_rank=6 min=31772 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=59785 max_load_reduction_pct=0.643666 moved_to_replicas=101384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=34 tokens_per_ep_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65190 max_load_reduction_pct=0.663268 moved_to_replicas=130411 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=36 tokens_per_ep_rank=[92882, 42255, 5, 65535, 6, 32776, 5, 28680] avg=32768.0000 max=92882 max_rank=0 min=5 min_rank=2 imbalance_ratio=2.834534 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=36 tokens_per_ep_rank=[92882, 42255, 5, 65535, 6, 32776, 5, 28680] avg=32768.0000 max=92882 max_rank=0 min=5 min_rank=2 imbalance_ratio=2.834534 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=36 tokens_per_ep_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] avg=32768.0000 max=33097 max_rank=6 min=31772 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=59785 max_load_reduction_pct=0.643666 moved_to_replicas=101384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=32 tokens_per_ep_rank=[32730, 32376, 33096, 32769, 32769, 33096, 32546, 32762] avg=32768.0000 max=33096 max_rank=2 min=32376 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32865 max_load_reduction_pct=0.498249 moved_to_replicas=65303 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=35 tokens_per_ep_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] avg=32768.0000 max=33096 max_rank=3 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97958 max_load_reduction_pct=0.747463 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=36 tokens_per_ep_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] avg=32768.0000 max=33097 max_rank=6 min=31772 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=59785 max_load_reduction_pct=0.643666 moved_to_replicas=101384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=37 tokens_per_ep_rank=[32763, 63594, 15, 1, 1954, 0, 65528, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=37 tokens_per_ep_rank=[32763, 63594, 15, 1, 1954, 0, 65528, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=36 tokens_per_ep_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] avg=32768.0000 max=33097 max_rank=6 min=31772 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=59785 max_load_reduction_pct=0.643666 moved_to_replicas=101384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=35 tokens_per_ep_rank=[10, 31483, 172, 131054, 32867, 11, 66547, 0] avg=32768.0000 max=131054 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=37 tokens_per_ep_rank=[32763, 63594, 15, 1, 1954, 0, 65528, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=37 tokens_per_ep_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=128123 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=37 tokens_per_ep_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=128123 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=33 tokens_per_ep_rank=[62244, 40, 65526, 13, 47, 3321, 98187, 32766] avg=32768.0000 max=98187 max_rank=6 min=13 min_rank=3 imbalance_ratio=2.996429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=36 tokens_per_ep_rank=[92882, 42255, 5, 65535, 6, 32776, 5, 28680] avg=32768.0000 max=92882 max_rank=0 min=5 min_rank=2 imbalance_ratio=2.834534 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=35 tokens_per_ep_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] avg=32768.0000 max=33096 max_rank=3 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97958 max_load_reduction_pct=0.747463 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=37 tokens_per_ep_rank=[32763, 63594, 15, 1, 1954, 0, 65528, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=37 tokens_per_ep_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=128123 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=37 tokens_per_ep_rank=[32763, 63594, 15, 1, 1954, 0, 65528, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=33 tokens_per_ep_rank=[33096, 32369, 33096, 32775, 32477, 32469, 33096, 32766] avg=32768.0000 max=33096 max_rank=0 min=32369 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65091 max_load_reduction_pct=0.662929 moved_to_replicas=126669 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=36 tokens_per_ep_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] avg=32768.0000 max=33097 max_rank=6 min=31772 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=59785 max_load_reduction_pct=0.643666 moved_to_replicas=101384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=38 tokens_per_ep_rank=[4448, 0, 65528, 32448, 32778, 32772, 32761, 61409] avg=32768.0000 max=65528 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=38 tokens_per_ep_rank=[4448, 0, 65528, 32448, 32778, 32772, 32761, 61409] avg=32768.0000 max=65528 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=37 tokens_per_ep_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=128123 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=37 tokens_per_ep_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=128123 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=36 tokens_per_ep_rank=[92882, 42255, 5, 65535, 6, 32776, 5, 28680] avg=32768.0000 max=92882 max_rank=0 min=5 min_rank=2 imbalance_ratio=2.834534 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=38 tokens_per_ep_rank=[4448, 0, 65528, 32448, 32778, 32772, 32761, 61409] avg=32768.0000 max=65528 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=38 tokens_per_ep_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=60745 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=38 tokens_per_ep_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=60745 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=34 tokens_per_ep_rank=[65879, 65534, 32419, 5, 21, 0, 0, 98286] avg=32768.0000 max=98286 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=37 tokens_per_ep_rank=[32763, 63594, 15, 1, 1954, 0, 65528, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=36 tokens_per_ep_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] avg=32768.0000 max=33097 max_rank=6 min=31772 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=59785 max_load_reduction_pct=0.643666 moved_to_replicas=101384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=38 tokens_per_ep_rank=[4448, 0, 65528, 32448, 32778, 32772, 32761, 61409] avg=32768.0000 max=65528 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=38 tokens_per_ep_rank=[4448, 0, 65528, 32448, 32778, 32772, 32761, 61409] avg=32768.0000 max=65528 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=38 tokens_per_ep_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=60745 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=34 tokens_per_ep_rank=[33096, 33096, 32419, 32767, 32480, 32762, 32428, 33096] avg=32768.0000 max=33096 max_rank=0 min=32419 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65190 max_load_reduction_pct=0.663268 moved_to_replicas=130411 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=39 tokens_per_ep_rank=[32810, 3, 32768, 32774, 9, 98264, 32771, 32745] avg=32768.0000 max=98264 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=37 tokens_per_ep_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=128123 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=38 tokens_per_ep_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=60745 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=39 tokens_per_ep_rank=[32810, 3, 32768, 32774, 9, 98264, 32771, 32745] avg=32768.0000 max=98264 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=38 tokens_per_ep_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=60745 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=37 tokens_per_ep_rank=[32763, 63594, 15, 1, 1954, 0, 65528, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=39 tokens_per_ep_rank=[32810, 3, 32768, 32774, 9, 98264, 32771, 32745] avg=32768.0000 max=98264 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=39 tokens_per_ep_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] avg=32768.0000 max=33096 max_rank=5 min=32415 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65168 max_load_reduction_pct=0.663193 moved_to_replicas=65168 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=39 tokens_per_ep_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] avg=32768.0000 max=33096 max_rank=5 min=32415 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65168 max_load_reduction_pct=0.663193 moved_to_replicas=65168 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=35 tokens_per_ep_rank=[10, 31483, 172, 131054, 32867, 11, 66547, 0] avg=32768.0000 max=131054 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=38 tokens_per_ep_rank=[4448, 0, 65528, 32448, 32778, 32772, 32761, 61409] avg=32768.0000 max=65528 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=37 tokens_per_ep_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=128123 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=39 tokens_per_ep_rank=[32810, 3, 32768, 32774, 9, 98264, 32771, 32745] avg=32768.0000 max=98264 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=39 tokens_per_ep_rank=[32810, 3, 32768, 32774, 9, 98264, 32771, 32745] avg=32768.0000 max=98264 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=39 tokens_per_ep_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] avg=32768.0000 max=33096 max_rank=5 min=32415 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65168 max_load_reduction_pct=0.663193 moved_to_replicas=65168 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=40 tokens_per_ep_rank=[98244, 65528, 3, 32769, 9, 32766, 32820, 5] avg=32768.0000 max=98244 max_rank=0 min=3 min_rank=2 imbalance_ratio=2.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=35 tokens_per_ep_rank=[32772, 32172, 32934, 33096, 32867, 32445, 33096, 32762] avg=32768.0000 max=33096 max_rank=3 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97958 max_load_reduction_pct=0.747463 moved_to_replicas=131409 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=38 tokens_per_ep_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=60745 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=40 tokens_per_ep_rank=[98244, 65528, 3, 32769, 9, 32766, 32820, 5] avg=32768.0000 max=98244 max_rank=0 min=3 min_rank=2 imbalance_ratio=2.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=39 tokens_per_ep_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] avg=32768.0000 max=33096 max_rank=5 min=32415 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65168 max_load_reduction_pct=0.663193 moved_to_replicas=65168 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=39 tokens_per_ep_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] avg=32768.0000 max=33096 max_rank=5 min=32415 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65168 max_load_reduction_pct=0.663193 moved_to_replicas=65168 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=38 tokens_per_ep_rank=[4448, 0, 65528, 32448, 32778, 32772, 32761, 61409] avg=32768.0000 max=65528 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=40 tokens_per_ep_rank=[98244, 65528, 3, 32769, 9, 32766, 32820, 5] avg=32768.0000 max=98244 max_rank=0 min=3 min_rank=2 imbalance_ratio=2.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=40 tokens_per_ep_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65148 max_load_reduction_pct=0.663124 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=40 tokens_per_ep_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65148 max_load_reduction_pct=0.663124 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=39 tokens_per_ep_rank=[32810, 3, 32768, 32774, 9, 98264, 32771, 32745] avg=32768.0000 max=98264 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=36 tokens_per_ep_rank=[92882, 42255, 5, 65535, 6, 32776, 5, 28680] avg=32768.0000 max=92882 max_rank=0 min=5 min_rank=2 imbalance_ratio=2.834534 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=38 tokens_per_ep_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=60745 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=40 tokens_per_ep_rank=[98244, 65528, 3, 32769, 9, 32766, 32820, 5] avg=32768.0000 max=98244 max_rank=0 min=3 min_rank=2 imbalance_ratio=2.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=40 tokens_per_ep_rank=[98244, 65528, 3, 32769, 9, 32766, 32820, 5] avg=32768.0000 max=98244 max_rank=0 min=3 min_rank=2 imbalance_ratio=2.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=40 tokens_per_ep_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65148 max_load_reduction_pct=0.663124 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=41 tokens_per_ep_rank=[15, 5, 59375, 38874, 4, 98232, 125, 65514] avg=32768.0000 max=98232 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=39 tokens_per_ep_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] avg=32768.0000 max=33096 max_rank=5 min=32415 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65168 max_load_reduction_pct=0.663193 moved_to_replicas=65168 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=36 tokens_per_ep_rank=[33096, 33095, 32767, 33096, 32445, 32776, 33097, 31772] avg=32768.0000 max=33097 max_rank=6 min=31772 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=59785 max_load_reduction_pct=0.643666 moved_to_replicas=101384 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=40 tokens_per_ep_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65148 max_load_reduction_pct=0.663124 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=41 tokens_per_ep_rank=[15, 5, 59375, 38874, 4, 98232, 125, 65514] avg=32768.0000 max=98232 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=40 tokens_per_ep_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65148 max_load_reduction_pct=0.663124 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=39 tokens_per_ep_rank=[32810, 3, 32768, 32774, 9, 98264, 32771, 32745] avg=32768.0000 max=98264 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=41 tokens_per_ep_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] avg=32768.0000 max=33098 max_rank=3 min=32180 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=129611 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=41 tokens_per_ep_rank=[15, 5, 59375, 38874, 4, 98232, 125, 65514] avg=32768.0000 max=98232 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=41 tokens_per_ep_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] avg=32768.0000 max=33098 max_rank=3 min=32180 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=129611 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=40 tokens_per_ep_rank=[98244, 65528, 3, 32769, 9, 32766, 32820, 5] avg=32768.0000 max=98244 max_rank=0 min=3 min_rank=2 imbalance_ratio=2.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=37 tokens_per_ep_rank=[32763, 63594, 15, 1, 1954, 0, 65528, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=39 tokens_per_ep_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] avg=32768.0000 max=33096 max_rank=5 min=32415 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65168 max_load_reduction_pct=0.663193 moved_to_replicas=65168 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=41 tokens_per_ep_rank=[15, 5, 59375, 38874, 4, 98232, 125, 65514] avg=32768.0000 max=98232 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=41 tokens_per_ep_rank=[15, 5, 59375, 38874, 4, 98232, 125, 65514] avg=32768.0000 max=98232 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=41 tokens_per_ep_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] avg=32768.0000 max=33098 max_rank=3 min=32180 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=129611 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=42 tokens_per_ep_rank=[90007, 2, 8, 32777, 61303, 65526, 32, 12489] avg=32768.0000 max=90007 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.746796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=40 tokens_per_ep_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65148 max_load_reduction_pct=0.663124 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=37 tokens_per_ep_rank=[32763, 33096, 32447, 32432, 32452, 32762, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=128123 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=41 tokens_per_ep_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] avg=32768.0000 max=33098 max_rank=3 min=32180 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=129611 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=42 tokens_per_ep_rank=[90007, 2, 8, 32777, 61303, 65526, 32, 12489] avg=32768.0000 max=90007 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.746796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=41 tokens_per_ep_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] avg=32768.0000 max=33098 max_rank=3 min=32180 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=129611 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=40 tokens_per_ep_rank=[98244, 65528, 3, 32769, 9, 32766, 32820, 5] avg=32768.0000 max=98244 max_rank=0 min=3 min_rank=2 imbalance_ratio=2.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=42 tokens_per_ep_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] avg=32768.0000 max=33098 max_rank=7 min=31756 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56909 max_load_reduction_pct=0.632273 moved_to_replicas=117548 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=42 tokens_per_ep_rank=[90007, 2, 8, 32777, 61303, 65526, 32, 12489] avg=32768.0000 max=90007 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.746796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=42 tokens_per_ep_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] avg=32768.0000 max=33098 max_rank=7 min=31756 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56909 max_load_reduction_pct=0.632273 moved_to_replicas=117548 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=41 tokens_per_ep_rank=[15, 5, 59375, 38874, 4, 98232, 125, 65514] avg=32768.0000 max=98232 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=38 tokens_per_ep_rank=[4448, 0, 65528, 32448, 32778, 32772, 32761, 61409] avg=32768.0000 max=65528 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=42 tokens_per_ep_rank=[90007, 2, 8, 32777, 61303, 65526, 32, 12489] avg=32768.0000 max=90007 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.746796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=40 tokens_per_ep_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65148 max_load_reduction_pct=0.663124 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=42 tokens_per_ep_rank=[90007, 2, 8, 32777, 61303, 65526, 32, 12489] avg=32768.0000 max=90007 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.746796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=42 tokens_per_ep_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] avg=32768.0000 max=33098 max_rank=7 min=31756 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56909 max_load_reduction_pct=0.632273 moved_to_replicas=117548 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=43 tokens_per_ep_rank=[5, 37631, 32727, 198, 33625, 33040, 30434, 94484] avg=32768.0000 max=94484 max_rank=7 min=5 min_rank=0 imbalance_ratio=2.883423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=41 tokens_per_ep_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] avg=32768.0000 max=33098 max_rank=3 min=32180 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=129611 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=38 tokens_per_ep_rank=[32761, 32432, 33096, 32448, 32778, 32772, 32761, 33096] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=60745 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=42 tokens_per_ep_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] avg=32768.0000 max=33098 max_rank=7 min=31756 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56909 max_load_reduction_pct=0.632273 moved_to_replicas=117548 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=43 tokens_per_ep_rank=[5, 37631, 32727, 198, 33625, 33040, 30434, 94484] avg=32768.0000 max=94484 max_rank=7 min=5 min_rank=0 imbalance_ratio=2.883423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=42 tokens_per_ep_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] avg=32768.0000 max=33098 max_rank=7 min=31756 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56909 max_load_reduction_pct=0.632273 moved_to_replicas=117548 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=43 tokens_per_ep_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] avg=32768.0000 max=33098 max_rank=7 min=31226 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=61386 max_load_reduction_pct=0.649697 moved_to_replicas=66452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=41 tokens_per_ep_rank=[15, 5, 59375, 38874, 4, 98232, 125, 65514] avg=32768.0000 max=98232 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=43 tokens_per_ep_rank=[5, 37631, 32727, 198, 33625, 33040, 30434, 94484] avg=32768.0000 max=94484 max_rank=7 min=5 min_rank=0 imbalance_ratio=2.883423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=43 tokens_per_ep_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] avg=32768.0000 max=33098 max_rank=7 min=31226 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=61386 max_load_reduction_pct=0.649697 moved_to_replicas=66452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=42 tokens_per_ep_rank=[90007, 2, 8, 32777, 61303, 65526, 32, 12489] avg=32768.0000 max=90007 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.746796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=39 tokens_per_ep_rank=[32810, 3, 32768, 32774, 9, 98264, 32771, 32745] avg=32768.0000 max=98264 max_rank=5 min=3 min_rank=1 imbalance_ratio=2.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=43 tokens_per_ep_rank=[5, 37631, 32727, 198, 33625, 33040, 30434, 94484] avg=32768.0000 max=94484 max_rank=7 min=5 min_rank=0 imbalance_ratio=2.883423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=41 tokens_per_ep_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] avg=32768.0000 max=33098 max_rank=3 min=32180 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=129611 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=43 tokens_per_ep_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] avg=32768.0000 max=33098 max_rank=7 min=31226 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=61386 max_load_reduction_pct=0.649697 moved_to_replicas=66452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=43 tokens_per_ep_rank=[5, 37631, 32727, 198, 33625, 33040, 30434, 94484] avg=32768.0000 max=94484 max_rank=7 min=5 min_rank=0 imbalance_ratio=2.883423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=44 tokens_per_ep_rank=[307, 32773, 97921, 33005, 15910, 49213, 183, 32832] avg=32768.0000 max=97921 max_rank=2 min=183 min_rank=6 imbalance_ratio=2.988312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=42 tokens_per_ep_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] avg=32768.0000 max=33098 max_rank=7 min=31756 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56909 max_load_reduction_pct=0.632273 moved_to_replicas=117548 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=39 tokens_per_ep_rank=[32810, 32765, 32768, 32774, 32415, 33096, 32771, 32745] avg=32768.0000 max=33096 max_rank=5 min=32415 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65168 max_load_reduction_pct=0.663193 moved_to_replicas=65168 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=43 tokens_per_ep_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] avg=32768.0000 max=33098 max_rank=7 min=31226 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=61386 max_load_reduction_pct=0.649697 moved_to_replicas=66452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=44 tokens_per_ep_rank=[307, 32773, 97921, 33005, 15910, 49213, 183, 32832] avg=32768.0000 max=97921 max_rank=2 min=183 min_rank=6 imbalance_ratio=2.988312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=43 tokens_per_ep_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] avg=32768.0000 max=33098 max_rank=7 min=31226 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=61386 max_load_reduction_pct=0.649697 moved_to_replicas=66452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=44 tokens_per_ep_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] avg=32768.0000 max=33096 max_rank=2 min=32027 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.662013 moved_to_replicas=80942 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=44 tokens_per_ep_rank=[307, 32773, 97921, 33005, 15910, 49213, 183, 32832] avg=32768.0000 max=97921 max_rank=2 min=183 min_rank=6 imbalance_ratio=2.988312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=42 tokens_per_ep_rank=[90007, 2, 8, 32777, 61303, 65526, 32, 12489] avg=32768.0000 max=90007 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.746796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=44 tokens_per_ep_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] avg=32768.0000 max=33096 max_rank=2 min=32027 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.662013 moved_to_replicas=80942 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=43 tokens_per_ep_rank=[5, 37631, 32727, 198, 33625, 33040, 30434, 94484] avg=32768.0000 max=94484 max_rank=7 min=5 min_rank=0 imbalance_ratio=2.883423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=44 tokens_per_ep_rank=[307, 32773, 97921, 33005, 15910, 49213, 183, 32832] avg=32768.0000 max=97921 max_rank=2 min=183 min_rank=6 imbalance_ratio=2.988312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=40 tokens_per_ep_rank=[98244, 65528, 3, 32769, 9, 32766, 32820, 5] avg=32768.0000 max=98244 max_rank=0 min=3 min_rank=2 imbalance_ratio=2.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=44 tokens_per_ep_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] avg=32768.0000 max=33096 max_rank=2 min=32027 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.662013 moved_to_replicas=80942 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=44 tokens_per_ep_rank=[307, 32773, 97921, 33005, 15910, 49213, 183, 32832] avg=32768.0000 max=97921 max_rank=2 min=183 min_rank=6 imbalance_ratio=2.988312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=42 tokens_per_ep_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] avg=32768.0000 max=33098 max_rank=7 min=31756 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56909 max_load_reduction_pct=0.632273 moved_to_replicas=117548 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=45 tokens_per_ep_rank=[10, 65530, 98290, 0, 115, 32763, 32663, 32773] avg=32768.0000 max=98290 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=43 tokens_per_ep_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] avg=32768.0000 max=33098 max_rank=7 min=31226 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=61386 max_load_reduction_pct=0.649697 moved_to_replicas=66452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=44 tokens_per_ep_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] avg=32768.0000 max=33096 max_rank=2 min=32027 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.662013 moved_to_replicas=80942 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=45 tokens_per_ep_rank=[10, 65530, 98290, 0, 115, 32763, 32663, 32773] avg=32768.0000 max=98290 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=40 tokens_per_ep_rank=[33096, 33096, 32765, 32769, 32441, 32766, 32820, 32391] avg=32768.0000 max=33096 max_rank=0 min=32391 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65148 max_load_reduction_pct=0.663124 moved_to_replicas=97580 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=44 tokens_per_ep_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] avg=32768.0000 max=33096 max_rank=2 min=32027 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.662013 moved_to_replicas=80942 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=45 tokens_per_ep_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65194 max_load_reduction_pct=0.663282 moved_to_replicas=97628 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=45 tokens_per_ep_rank=[10, 65530, 98290, 0, 115, 32763, 32663, 32773] avg=32768.0000 max=98290 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=43 tokens_per_ep_rank=[5, 37631, 32727, 198, 33625, 33040, 30434, 94484] avg=32768.0000 max=94484 max_rank=7 min=5 min_rank=0 imbalance_ratio=2.883423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=45 tokens_per_ep_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65194 max_load_reduction_pct=0.663282 moved_to_replicas=97628 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=44 tokens_per_ep_rank=[307, 32773, 97921, 33005, 15910, 49213, 183, 32832] avg=32768.0000 max=97921 max_rank=2 min=183 min_rank=6 imbalance_ratio=2.988312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=45 tokens_per_ep_rank=[10, 65530, 98290, 0, 115, 32763, 32663, 32773] avg=32768.0000 max=98290 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=45 tokens_per_ep_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65194 max_load_reduction_pct=0.663282 moved_to_replicas=97628 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=41 tokens_per_ep_rank=[15, 5, 59375, 38874, 4, 98232, 125, 65514] avg=32768.0000 max=98232 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=45 tokens_per_ep_rank=[10, 65530, 98290, 0, 115, 32763, 32663, 32773] avg=32768.0000 max=98290 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=46 tokens_per_ep_rank=[34851, 10, 34614, 2, 9, 65838, 94059, 32761] avg=32768.0000 max=94059 max_rank=6 min=2 min_rank=3 imbalance_ratio=2.870453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=43 tokens_per_ep_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] avg=32768.0000 max=33098 max_rank=7 min=31226 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=61386 max_load_reduction_pct=0.649697 moved_to_replicas=66452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=46 tokens_per_ep_rank=[34851, 10, 34614, 2, 9, 65838, 94059, 32761] avg=32768.0000 max=94059 max_rank=6 min=2 min_rank=3 imbalance_ratio=2.870453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=44 tokens_per_ep_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] avg=32768.0000 max=33096 max_rank=2 min=32027 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.662013 moved_to_replicas=80942 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=45 tokens_per_ep_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65194 max_load_reduction_pct=0.663282 moved_to_replicas=97628 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=41 tokens_per_ep_rank=[32433, 32379, 33096, 33098, 32766, 33096, 32180, 33096] avg=32768.0000 max=33098 max_rank=3 min=32180 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65134 max_load_reduction_pct=0.663063 moved_to_replicas=129611 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=45 tokens_per_ep_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65194 max_load_reduction_pct=0.663282 moved_to_replicas=97628 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=46 tokens_per_ep_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=31483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60963 max_load_reduction_pct=0.648136 moved_to_replicas=96978 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=46 tokens_per_ep_rank=[34851, 10, 34614, 2, 9, 65838, 94059, 32761] avg=32768.0000 max=94059 max_rank=6 min=2 min_rank=3 imbalance_ratio=2.870453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=44 tokens_per_ep_rank=[307, 32773, 97921, 33005, 15910, 49213, 183, 32832] avg=32768.0000 max=97921 max_rank=2 min=183 min_rank=6 imbalance_ratio=2.988312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=46 tokens_per_ep_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=31483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60963 max_load_reduction_pct=0.648136 moved_to_replicas=96978 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=46 tokens_per_ep_rank=[34851, 10, 34614, 2, 9, 65838, 94059, 32761] avg=32768.0000 max=94059 max_rank=6 min=2 min_rank=3 imbalance_ratio=2.870453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=45 tokens_per_ep_rank=[10, 65530, 98290, 0, 115, 32763, 32663, 32773] avg=32768.0000 max=98290 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=46 tokens_per_ep_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=31483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60963 max_load_reduction_pct=0.648136 moved_to_replicas=96978 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=46 tokens_per_ep_rank=[34851, 10, 34614, 2, 9, 65838, 94059, 32761] avg=32768.0000 max=94059 max_rank=6 min=2 min_rank=3 imbalance_ratio=2.870453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=42 tokens_per_ep_rank=[90007, 2, 8, 32777, 61303, 65526, 32, 12489] avg=32768.0000 max=90007 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.746796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=44 tokens_per_ep_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] avg=32768.0000 max=33096 max_rank=2 min=32027 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.662013 moved_to_replicas=80942 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=47 tokens_per_ep_rank=[10472, 25890, 32768, 65528, 32766, 38, 32761, 61921] avg=32768.0000 max=65528 max_rank=3 min=38 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=47 tokens_per_ep_rank=[10472, 25890, 32768, 65528, 32766, 38, 32761, 61921] avg=32768.0000 max=65528 max_rank=3 min=38 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=46 tokens_per_ep_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=31483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60963 max_load_reduction_pct=0.648136 moved_to_replicas=96978 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=45 tokens_per_ep_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65194 max_load_reduction_pct=0.663282 moved_to_replicas=97628 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=46 tokens_per_ep_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=31483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60963 max_load_reduction_pct=0.648136 moved_to_replicas=96978 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=42 tokens_per_ep_rank=[33096, 32764, 31756, 32777, 33095, 33096, 32462, 33098] avg=32768.0000 max=33098 max_rank=7 min=31756 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=56909 max_load_reduction_pct=0.632273 moved_to_replicas=117548 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=47 tokens_per_ep_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] avg=32768.0000 max=33096 max_rank=3 min=32092 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=61257 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=47 tokens_per_ep_rank=[10472, 25890, 32768, 65528, 32766, 38, 32761, 61921] avg=32768.0000 max=65528 max_rank=3 min=38 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=47 tokens_per_ep_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] avg=32768.0000 max=33096 max_rank=3 min=32092 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=61257 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=45 tokens_per_ep_rank=[10, 65530, 98290, 0, 115, 32763, 32663, 32773] avg=32768.0000 max=98290 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=47 tokens_per_ep_rank=[10472, 25890, 32768, 65528, 32766, 38, 32761, 61921] avg=32768.0000 max=65528 max_rank=3 min=38 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=47 tokens_per_ep_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] avg=32768.0000 max=33096 max_rank=3 min=32092 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=61257 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=46 tokens_per_ep_rank=[34851, 10, 34614, 2, 9, 65838, 94059, 32761] avg=32768.0000 max=94059 max_rank=6 min=2 min_rank=3 imbalance_ratio=2.870453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=47 tokens_per_ep_rank=[10472, 25890, 32768, 65528, 32766, 38, 32761, 61921] avg=32768.0000 max=65528 max_rank=3 min=38 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=43 tokens_per_ep_rank=[5, 37631, 32727, 198, 33625, 33040, 30434, 94484] avg=32768.0000 max=94484 max_rank=7 min=5 min_rank=0 imbalance_ratio=2.883423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=45 tokens_per_ep_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65194 max_load_reduction_pct=0.663282 moved_to_replicas=97628 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=48 tokens_per_ep_rank=[98280, 32843, 10, 1385, 5, 90656, 4, 38961] avg=32768.0000 max=98280 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=47 tokens_per_ep_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] avg=32768.0000 max=33096 max_rank=3 min=32092 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=61257 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=48 tokens_per_ep_rank=[98280, 32843, 10, 1385, 5, 90656, 4, 38961] avg=32768.0000 max=98280 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=46 tokens_per_ep_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=31483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60963 max_load_reduction_pct=0.648136 moved_to_replicas=96978 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=47 tokens_per_ep_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] avg=32768.0000 max=33096 max_rank=3 min=32092 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=61257 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=43 tokens_per_ep_rank=[32767, 33094, 32727, 33096, 33096, 33040, 31226, 33098] avg=32768.0000 max=33098 max_rank=7 min=31226 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=61386 max_load_reduction_pct=0.649697 moved_to_replicas=66452 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=48 tokens_per_ep_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=32048 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65184 max_load_reduction_pct=0.663248 moved_to_replicas=128609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=48 tokens_per_ep_rank=[98280, 32843, 10, 1385, 5, 90656, 4, 38961] avg=32768.0000 max=98280 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=48 tokens_per_ep_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=32048 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65184 max_load_reduction_pct=0.663248 moved_to_replicas=128609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=46 tokens_per_ep_rank=[34851, 10, 34614, 2, 9, 65838, 94059, 32761] avg=32768.0000 max=94059 max_rank=6 min=2 min_rank=3 imbalance_ratio=2.870453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=48 tokens_per_ep_rank=[98280, 32843, 10, 1385, 5, 90656, 4, 38961] avg=32768.0000 max=98280 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=47 tokens_per_ep_rank=[10472, 25890, 32768, 65528, 32766, 38, 32761, 61921] avg=32768.0000 max=65528 max_rank=3 min=38 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=48 tokens_per_ep_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=32048 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65184 max_load_reduction_pct=0.663248 moved_to_replicas=128609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=48 tokens_per_ep_rank=[98280, 32843, 10, 1385, 5, 90656, 4, 38961] avg=32768.0000 max=98280 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=46 tokens_per_ep_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=31483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60963 max_load_reduction_pct=0.648136 moved_to_replicas=96978 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=44 tokens_per_ep_rank=[307, 32773, 97921, 33005, 15910, 49213, 183, 32832] avg=32768.0000 max=97921 max_rank=2 min=183 min_rank=6 imbalance_ratio=2.988312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=48 tokens_per_ep_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=32048 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65184 max_load_reduction_pct=0.663248 moved_to_replicas=128609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=47 tokens_per_ep_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] avg=32768.0000 max=33096 max_rank=3 min=32092 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=61257 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=48 tokens_per_ep_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=32048 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65184 max_load_reduction_pct=0.663248 moved_to_replicas=128609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=44 tokens_per_ep_rank=[32370, 32773, 33096, 33005, 32027, 33096, 32945, 32832] avg=32768.0000 max=33096 max_rank=2 min=32027 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.662013 moved_to_replicas=80942 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=47 tokens_per_ep_rank=[10472, 25890, 32768, 65528, 32766, 38, 32761, 61921] avg=32768.0000 max=65528 max_rank=3 min=38 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=48 tokens_per_ep_rank=[98280, 32843, 10, 1385, 5, 90656, 4, 38961] avg=32768.0000 max=98280 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=47 tokens_per_ep_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] avg=32768.0000 max=33096 max_rank=3 min=32092 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=61257 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=45 tokens_per_ep_rank=[10, 65530, 98290, 0, 115, 32763, 32663, 32773] avg=32768.0000 max=98290 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=48 tokens_per_ep_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=32048 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65184 max_load_reduction_pct=0.663248 moved_to_replicas=128609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=45 tokens_per_ep_rank=[32442, 33096, 33096, 32762, 32549, 32763, 32663, 32773] avg=32768.0000 max=33096 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65194 max_load_reduction_pct=0.663282 moved_to_replicas=97628 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=48 tokens_per_ep_rank=[98280, 32843, 10, 1385, 5, 90656, 4, 38961] avg=32768.0000 max=98280 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=46 tokens_per_ep_rank=[34851, 10, 34614, 2, 9, 65838, 94059, 32761] avg=32768.0000 max=94059 max_rank=6 min=2 min_rank=3 imbalance_ratio=2.870453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=48 tokens_per_ep_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=32048 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65184 max_load_reduction_pct=0.663248 moved_to_replicas=128609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=46 tokens_per_ep_rank=[33096, 32752, 33096, 32764, 31483, 33096, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=31483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=60963 max_load_reduction_pct=0.648136 moved_to_replicas=96978 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=47 tokens_per_ep_rank=[10472, 25890, 32768, 65528, 32766, 38, 32761, 61921] avg=32768.0000 max=65528 max_rank=3 min=38 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=47 tokens_per_ep_rank=[33095, 32092, 32768, 33096, 32766, 32470, 32761, 33096] avg=32768.0000 max=33096 max_rank=3 min=32092 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=61257 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=48 tokens_per_ep_rank=[98280, 32843, 10, 1385, 5, 90656, 4, 38961] avg=32768.0000 max=98280 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=48 tokens_per_ep_rank=[33096, 32843, 32772, 32048, 32427, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=0 min=32048 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65184 max_load_reduction_pct=0.663248 moved_to_replicas=128609 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=48 total_expected_recv=1568364 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379904 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=48 total_expected_recv=1567310 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379776 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=48 total_expected_recv=1578374 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=370048 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=48 total_expected_recv=1573418 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376448 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=48 total_expected_recv=1578615 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368768 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=48 total_expected_recv=1567372 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=382080 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=48 total_expected_recv=1575982 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=371072 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=48 total_expected_recv=1573477 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375808 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 15:47:04] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 920.9 | throughput per GPU (TFLOP/s/GPU): 96.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.852440E+00 | loss scale: 1.0 | grad norm: 3.293 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31094 expected_overflow=0 exact_recv_per_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=31094 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=31709 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=31859 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=31709 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=30948 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=30948 actual_valid_recv=31360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=31859 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32314 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=31253 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32314 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=31253 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32041 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32134 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32041 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32134 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=31775 expected_overflow=0 exact_recv_per_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=31775 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32307 expected_overflow=0 exact_recv_per_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32307 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=31005 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=31005 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32304 expected_overflow=0 exact_recv_per_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32369 expected_overflow=0 exact_recv_per_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=31795 expected_overflow=0 exact_recv_per_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32304 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32369 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=31795 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=31169 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=31169 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=31289 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=31289 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32279 expected_overflow=0 exact_recv_per_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32279 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=31308 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=31721 expected_overflow=0 exact_recv_per_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=31721 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=31308 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32548 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32548 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=31833 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=31833 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=31026 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=31026 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=40960 expected_recv=31844 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=40960 expected_recv=31844 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32057 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32057 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32004 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=40960 expected_recv=31914 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32004 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=40960 expected_recv=31914 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=31269 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=31269 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=31478 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=31478 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32410 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32410 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=40960 expected_recv=31456 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=40960 expected_recv=31456 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=31661 expected_overflow=0 exact_recv_per_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=31661 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=40960 expected_recv=31958 expected_overflow=0 exact_recv_per_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=40960 expected_recv=31958 actual_valid_recv=32256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32331 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32331 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32408 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32408 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=40960 expected_recv=31783 expected_overflow=0 exact_recv_per_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=40960 expected_recv=31783 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32254 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32254 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=40960 expected_recv=31237 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=40960 expected_recv=31237 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32105 expected_overflow=0 exact_recv_per_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32105 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32550 expected_overflow=0 exact_recv_per_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=40960 expected_recv=31130 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=40960 expected_recv=31130 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=40960 expected_recv=31615 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=40960 expected_recv=31615 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32354 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32354 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32131 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=40960 expected_recv=31684 expected_overflow=0 exact_recv_per_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32550 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=40960 expected_recv=31684 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32510 expected_overflow=0 exact_recv_per_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32510 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32131 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=40960 expected_recv=31735 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=40960 expected_recv=31735 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32124 expected_overflow=0 exact_recv_per_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32124 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=40960 expected_recv=31908 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=40960 expected_recv=31908 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=40960 expected_recv=31439 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=40960 expected_recv=31439 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32020 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32020 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=40960 expected_recv=31463 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=40960 expected_recv=31463 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32113 expected_overflow=0 exact_recv_per_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32113 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32490 expected_overflow=0 exact_recv_per_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32490 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=40960 expected_recv=31834 expected_overflow=0 exact_recv_per_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32348 expected_overflow=0 exact_recv_per_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=40960 expected_recv=31834 actual_valid_recv=32128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32348 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=23 calls=1 budget=40960 expected_recv=31881 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=23 calls=1 budget=40960 expected_recv=31881 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32361 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=23 calls=1 budget=40960 expected_recv=31911 expected_overflow=0 exact_recv_per_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=23 calls=1 budget=40960 expected_recv=31911 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32361 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32340 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32340 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32225 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32225 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32402 expected_overflow=0 exact_recv_per_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32378 expected_overflow=0 exact_recv_per_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32402 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32378 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32058 expected_overflow=0 exact_recv_per_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=25 calls=1 budget=40960 expected_recv=31783 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=25 calls=1 budget=40960 expected_recv=31783 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=26 calls=1 budget=40960 expected_recv=31834 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32910 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=26 calls=1 budget=40960 expected_recv=31834 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32910 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32371 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32371 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32374 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32374 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32368 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=29 calls=1 budget=40960 expected_recv=31757 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32368 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=29 calls=1 budget=40960 expected_recv=31757 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32058 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32524 expected_overflow=0 exact_recv_per_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33099 expected_overflow=0 exact_recv_per_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32524 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33099 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33092 expected_overflow=0 exact_recv_per_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33092 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33055 expected_overflow=0 exact_recv_per_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32260 expected_overflow=0 exact_recv_per_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33055 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32260 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=28 calls=1 budget=40960 expected_recv=31814 expected_overflow=0 exact_recv_per_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=28 calls=1 budget=40960 expected_recv=31814 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=26 calls=1 budget=40960 expected_recv=31147 expected_overflow=0 exact_recv_per_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=26 calls=1 budget=40960 expected_recv=31147 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=29 calls=1 budget=40960 expected_recv=31823 expected_overflow=0 exact_recv_per_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=29 calls=1 budget=40960 expected_recv=31823 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=27 calls=1 budget=40960 expected_recv=31075 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=27 calls=1 budget=40960 expected_recv=31075 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32111 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32111 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=31 calls=1 budget=40960 expected_recv=31756 expected_overflow=0 exact_recv_per_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32350 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32350 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32410 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32410 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=31 calls=1 budget=40960 expected_recv=31756 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32978 expected_overflow=0 exact_recv_per_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32978 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=32 calls=1 budget=40960 expected_recv=31446 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=32 calls=1 budget=40960 expected_recv=31446 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32118 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32118 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=33 calls=1 budget=40960 expected_recv=31456 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=33 calls=1 budget=40960 expected_recv=31456 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32382 expected_overflow=0 exact_recv_per_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32382 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32295 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32295 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32141 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32141 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32154 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32154 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32253 expected_overflow=0 exact_recv_per_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32253 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32398 expected_overflow=0 exact_recv_per_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32398 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32059 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32059 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=40 calls=1 budget=40960 expected_recv=31974 expected_overflow=0 exact_recv_per_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=40 calls=1 budget=40960 expected_recv=31974 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32071 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32071 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32416 expected_overflow=0 exact_recv_per_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32416 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=43 calls=1 budget=40960 expected_recv=31453 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=42 calls=1 budget=40960 expected_recv=31455 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=43 calls=1 budget=40960 expected_recv=31453 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=42 calls=1 budget=40960 expected_recv=31455 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=43 calls=1 budget=40960 expected_recv=31766 expected_overflow=0 exact_recv_per_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32061 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32061 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=43 calls=1 budget=40960 expected_recv=31766 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33088 expected_overflow=0 exact_recv_per_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33088 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=42 calls=1 budget=40960 expected_recv=31927 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=42 calls=1 budget=40960 expected_recv=31927 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32128 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32128 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32052 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32052 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=46 calls=1 budget=40960 expected_recv=31795 expected_overflow=0 exact_recv_per_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32227 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=46 calls=1 budget=40960 expected_recv=31795 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32227 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32096 expected_overflow=0 exact_recv_per_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32096 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[23213, 26981, 34503, 46652, 41308, 23620, 27365, 38502] avg=32768.0000 max=46652 max_rank=3 min=23213 min_rank=0 imbalance_ratio=1.423706 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=47 calls=1 budget=40960 expected_recv=31841 expected_overflow=0 exact_recv_per_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=47 calls=1 budget=40960 expected_recv=31841 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[23991, 23710, 38376, 40799, 41458, 26378, 31101, 36331] avg=32768.0000 max=41458 max_rank=4 min=23710 min_rank=1 imbalance_ratio=1.265198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[33095, 32072, 33096, 33097, 33096, 33053, 31538, 33097] avg=32768.0000 max=33097 max_rank=3 min=31538 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=8361 max_load_reduction_pct=0.201674 moved_to_replicas=24580 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[81533, 3142, 5631, 33605, 33287, 39479, 48312, 17155] avg=32768.0000 max=81533 max_rank=0 min=3142 min_rank=1 imbalance_ratio=2.488190 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33095, 30472, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=48436 max_load_reduction_pct=0.594066 moved_to_replicas=70736 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[72311, 19292, 17555, 24335, 60679, 30948, 29074, 7950] avg=32768.0000 max=72311 max_rank=0 min=7950 min_rank=7 imbalance_ratio=2.206757 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 33095, 30948, 32620, 33096] avg=32768.0000 max=33097 max_rank=2 min=30948 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39214 max_load_reduction_pct=0.542296 moved_to_replicas=66798 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[233, 31323, 32901, 47801, 1506, 32134, 38030, 78216] avg=32768.0000 max=78216 max_rank=7 min=233 min_rank=0 imbalance_ratio=2.386963 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[33096, 32041, 32901, 33096, 32683, 32134, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=32041 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45119 max_load_reduction_pct=0.576851 moved_to_replicas=64759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[84042, 22325, 20742, 58993, 64188, 1176, 7346, 3332] avg=32768.0000 max=84042 max_rank=0 min=1176 min_rank=5 imbalance_ratio=2.564758 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[33096, 33096, 30472, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50946 max_load_reduction_pct=0.606197 moved_to_replicas=107935 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[16794, 12557, 33621, 53245, 56103, 56751, 32561, 512] avg=32768.0000 max=56751 max_rank=5 min=512 min_rank=7 imbalance_ratio=1.731903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33097, 33097, 33096, 33096, 33095, 33097, 32561, 31005] avg=32768.0000 max=33097 max_rank=0 min=31005 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=23654 max_load_reduction_pct=0.416803 moved_to_replicas=67336 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[33267, 3940, 9166, 49966, 531, 57249, 43181, 64844] avg=32768.0000 max=64844 max_rank=7 min=531 min_rank=4 imbalance_ratio=1.978882 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[33096, 31289, 33096, 33096, 32279, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31289 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31748 max_load_reduction_pct=0.489606 moved_to_replicas=83027 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[44833, 103, 54028, 40186, 32809, 23268, 1, 66916] avg=32768.0000 max=66916 max_rank=7 min=1 min_rank=6 imbalance_ratio=2.042114 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33095, 33097, 33095, 32809, 31308, 32548, 33096] avg=32768.0000 max=33097 max_rank=2 min=31308 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=33819 max_load_reduction_pct=0.505395 moved_to_replicas=73579 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[39950, 65726, 32758, 6, 32796, 29996, 25979, 34933] avg=32768.0000 max=65726 max_rank=1 min=6 min_rank=3 imbalance_ratio=2.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33095, 33096, 32758, 32636, 32796, 31833, 32834, 33096] avg=32768.0000 max=33096 max_rank=1 min=31833 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32630 max_load_reduction_pct=0.496455 moved_to_replicas=41321 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[21452, 41917, 61798, 2324, 21063, 21758, 32770, 59062] avg=32768.0000 max=61798 max_rank=2 min=2324 min_rank=3 imbalance_ratio=1.885925 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[33096, 33097, 33096, 31026, 33096, 32867, 32770, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=28701 max_load_reduction_pct=0.464433 moved_to_replicas=63489 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[63950, 63906, 32763, 33711, 4829, 31844, 31136, 5] avg=32768.0000 max=63950 max_rank=0 min=5 min_rank=7 imbalance_ratio=1.951599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[33096, 33096, 32763, 33096, 33096, 31844, 32057, 33096] avg=32768.0000 max=33096 max_rank=0 min=31844 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30854 max_load_reduction_pct=0.482471 moved_to_replicas=62279 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[18439, 25812, 65007, 32747, 33451, 44148, 42537, 3] avg=32768.0000 max=65007 max_rank=2 min=3 min_rank=7 imbalance_ratio=1.983856 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32004, 33096, 33096, 32747, 33096, 33095, 33096, 31914] avg=32768.0000 max=33096 max_rank=1 min=31914 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31911 max_load_reduction_pct=0.490886 moved_to_replicas=52759 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[47904, 54857, 76842, 32331, 2, 1692, 32254, 16262] avg=32768.0000 max=76842 max_rank=2 min=2 min_rank=4 imbalance_ratio=2.345032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[33096, 33096, 33095, 32331, 32767, 33097, 32254, 32408] avg=32768.0000 max=33097 max_rank=5 min=32254 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43745 max_load_reduction_pct=0.569285 moved_to_replicas=80315 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[32723, 65588, 56380, 36327, 7320, 211, 4743, 58852] avg=32768.0000 max=65588 max_rank=1 min=211 min_rank=5 imbalance_ratio=2.001587 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[32723, 33096, 33097, 33095, 31237, 32703, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=31237 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32491 max_load_reduction_pct=0.495380 moved_to_replicas=84763 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32772, 65006, 27661, 8, 40053, 2358, 94284, 2] avg=32768.0000 max=94284 max_rank=6 min=2 min_rank=7 imbalance_ratio=2.877319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32772, 33096, 31130, 33098, 33094, 33096, 33096, 32762] avg=32768.0000 max=33098 max_rank=3 min=31130 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=61186 max_load_reduction_pct=0.648954 moved_to_replicas=100055 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[27321, 33346, 32808, 32773, 32564, 39244, 5, 64083] avg=32768.0000 max=64083 max_rank=7 min=5 min_rank=6 imbalance_ratio=1.955658 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[33095, 33096, 32808, 32773, 32564, 33097, 31615, 33096] avg=32768.0000 max=33097 max_rank=5 min=31615 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=30986 max_load_reduction_pct=0.483529 moved_to_replicas=37385 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[2, 7, 32752, 98111, 27, 101580, 29578, 87] avg=32768.0000 max=101580 max_rank=5 min=2 min_rank=0 imbalance_ratio=3.099976 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[32767, 32354, 32752, 33096, 33096, 33096, 32131, 32852] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=68484 max_load_reduction_pct=0.674188 moved_to_replicas=133499 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[62534, 11953, 31735, 32770, 47516, 32768, 32757, 10111] avg=32768.0000 max=62534 max_rank=0 min=10111 min_rank=7 imbalance_ratio=1.908386 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[33096, 32824, 31735, 32770, 33096, 32768, 32757, 33098] avg=32768.0000 max=33098 max_rank=7 min=31735 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=29436 max_load_reduction_pct=0.470720 moved_to_replicas=43858 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[6, 0, 32802, 51876, 32772, 31908, 32728, 80052] avg=32768.0000 max=80052 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.442993 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[32980, 32764, 32802, 33095, 32772, 31908, 32728, 33095] avg=32768.0000 max=33095 max_rank=3 min=31908 min_rank=5 imbalance_ratio=1.009979 max_load_reduction=46957 max_load_reduction_pct=0.586581 moved_to_replicas=65736 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[32770, 44511, 31463, 35028, 24057, 0, 35201, 59114] avg=32768.0000 max=59114 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.804016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[32770, 33096, 31463, 33096, 33096, 32431, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31463 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=26018 max_load_reduction_pct=0.440133 moved_to_replicas=41470 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[22, 45, 3, 6, 71070, 92711, 65522, 32765] avg=32768.0000 max=92711 max_rank=5 min=3 min_rank=2 imbalance_ratio=2.829315 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[32787, 33098, 32768, 31439, 33096, 33095, 33096, 32765] avg=32768.0000 max=33098 max_rank=1 min=31439 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=59613 max_load_reduction_pct=0.642998 moved_to_replicas=130015 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[98296, 4, 2, 9, 65574, 32769, 32820, 32670] avg=32768.0000 max=98296 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999756 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[33096, 32439, 32767, 32487, 33096, 32769, 32820, 32670] avg=32768.0000 max=33096 max_rank=0 min=32439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=97678 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[32774, 65534, 65535, 32740, 35396, 17, 30061, 87] avg=32768.0000 max=65535 max_rank=2 min=17 min_rank=5 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[32774, 33096, 33096, 32740, 33096, 32456, 32361, 32525] avg=32768.0000 max=33096 max_rank=1 min=32361 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=67177 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[72, 223, 7, 32770, 32770, 71341, 65535, 59426] avg=32768.0000 max=71341 max_rank=5 min=7 min_rank=2 imbalance_ratio=2.177155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[31881, 32662, 32772, 32770, 32770, 33096, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31881 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38244 max_load_reduction_pct=0.536073 moved_to_replicas=97014 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=25 tokens_per_ep_rank=[32340, 35717, 95996, 65284, 0, 2, 32768, 37] avg=32768.0000 max=95996 max_rank=2 min=0 min_rank=4 imbalance_ratio=2.929565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=25 tokens_per_ep_rank=[32340, 33096, 33096, 33096, 32765, 32758, 32768, 32225] avg=32768.0000 max=33096 max_rank=1 min=32225 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62900 max_load_reduction_pct=0.655236 moved_to_replicas=97709 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=26 tokens_per_ep_rank=[22, 54163, 32766, 65532, 32770, 5, 63342, 13544] avg=32768.0000 max=65532 max_rank=3 min=5 min_rank=5 imbalance_ratio=1.999878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=26 tokens_per_ep_rank=[31783, 33098, 32766, 33096, 32770, 32441, 33096, 33094] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=83749 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 1, 99166, 28785, 35746, 3] avg=32768.0000 max=99166 max_rank=4 min=1 min_rank=3 imbalance_ratio=3.026306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=27 tokens_per_ep_rank=[32772, 32767, 32904, 32765, 33096, 32910, 33096, 31834] avg=32768.0000 max=33096 max_rank=4 min=31834 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=66070 max_load_reduction_pct=0.666257 moved_to_replicas=68720 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=28 tokens_per_ep_rank=[1, 34486, 3, 87138, 76955, 32463, 2, 31096] avg=32768.0000 max=87138 max_rank=3 min=1 min_rank=0 imbalance_ratio=2.659241 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=28 tokens_per_ep_rank=[32766, 33096, 32768, 33097, 33094, 32463, 32374, 32486] avg=32768.0000 max=33097 max_rank=3 min=32374 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=54041 max_load_reduction_pct=0.620177 moved_to_replicas=99291 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=29 tokens_per_ep_rank=[31903, 32577, 65533, 32781, 33564, 32759, 31860, 1167] avg=32768.0000 max=65533 max_rank=2 min=1167 min_rank=7 imbalance_ratio=1.999908 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=29 tokens_per_ep_rank=[32371, 32577, 33096, 32781, 33096, 32759, 32368, 33096] avg=32768.0000 max=33096 max_rank=2 min=32368 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=32905 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=30 tokens_per_ep_rank=[8046, 65782, 0, 32771, 32795, 2, 89976, 32772] avg=32768.0000 max=89976 max_rank=6 min=0 min_rank=2 imbalance_ratio=2.745850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32771, 32795, 31757, 33096, 32772] avg=32768.0000 max=33096 max_rank=0 min=31757 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=56880 max_load_reduction_pct=0.632169 moved_to_replicas=89566 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=31 tokens_per_ep_rank=[6, 10, 98274, 32763, 32791, 32766, 0, 65534] avg=32768.0000 max=98274 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=31 tokens_per_ep_rank=[32419, 32448, 33096, 32763, 32791, 32766, 32765, 33096] avg=32768.0000 max=33096 max_rank=2 min=32419 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65178 max_load_reduction_pct=0.663227 moved_to_replicas=97616 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=32 tokens_per_ep_rank=[2, 32653, 5, 65523, 65441, 32988, 32766, 32766] avg=32768.0000 max=65523 max_rank=3 min=2 min_rank=0 imbalance_ratio=1.999603 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=32 tokens_per_ep_rank=[32429, 32653, 32350, 33096, 33096, 32988, 32766, 32766] avg=32768.0000 max=33096 max_rank=3 min=32350 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=64772 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=33 tokens_per_ep_rank=[4, 25189, 65562, 2, 32766, 0, 98304, 40317] avg=32768.0000 max=98304 max_rank=6 min=0 min_rank=5 imbalance_ratio=3.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=33 tokens_per_ep_rank=[32470, 32410, 33096, 32445, 32766, 32765, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65208 max_load_reduction_pct=0.663330 moved_to_replicas=104895 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=34 tokens_per_ep_rank=[33304, 63208, 98302, 2, 946, 526, 0, 65856] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32443, 31456, 33096, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=31456 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=128286 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=35 tokens_per_ep_rank=[32729, 4, 32772, 98569, 1, 9, 65531, 32529] avg=32768.0000 max=98569 max_rank=3 min=1 min_rank=4 imbalance_ratio=3.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=35 tokens_per_ep_rank=[32729, 32712, 32772, 33096, 32766, 32444, 33096, 32529] avg=32768.0000 max=33096 max_rank=3 min=32444 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65473 max_load_reduction_pct=0.664235 moved_to_replicas=97908 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=36 tokens_per_ep_rank=[65534, 32722, 27, 32649, 11, 65536, 32768, 32897] avg=32768.0000 max=65536 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32722, 32465, 32649, 32451, 33096, 32768, 32897] avg=32768.0000 max=33096 max_rank=0 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=64878 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=37 tokens_per_ep_rank=[31400, 32770, 32771, 32766, 65530, 0, 34132, 32775] avg=32768.0000 max=65530 max_rank=4 min=0 min_rank=5 imbalance_ratio=1.999817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=37 tokens_per_ep_rank=[32436, 32770, 32771, 32766, 33096, 32434, 33096, 32775] avg=32768.0000 max=33096 max_rank=4 min=32434 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=33470 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=38 tokens_per_ep_rank=[32769, 0, 98302, 32765, 32772, 32777, 3, 32756] avg=32768.0000 max=98302 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=38 tokens_per_ep_rank=[32769, 32765, 33096, 32765, 32772, 32777, 32444, 32756] avg=32768.0000 max=33096 max_rank=2 min=32444 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65206 max_load_reduction_pct=0.663323 moved_to_replicas=65206 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=39 tokens_per_ep_rank=[0, 2, 65535, 5, 32740, 65095, 65534, 33233] avg=32768.0000 max=65535 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=39 tokens_per_ep_rank=[32439, 32440, 33096, 32141, 32740, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=32141 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97013 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=40 tokens_per_ep_rank=[65250, 65191, 0, 65534, 32770, 631, 32768, 0] avg=32768.0000 max=65534 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32438, 33096, 32770, 32726, 32768, 32154] avg=32768.0000 max=33096 max_rank=0 min=32154 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=96687 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=41 tokens_per_ep_rank=[41093, 2, 65165, 32451, 1, 57890, 7, 65535] avg=32768.0000 max=65535 max_rank=7 min=1 min_rank=4 imbalance_ratio=1.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=41 tokens_per_ep_rank=[33096, 32071, 33096, 32451, 32440, 33096, 32798, 33096] avg=32768.0000 max=33096 max_rank=0 min=32071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=97299 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=42 tokens_per_ep_rank=[63711, 32768, 1811, 65511, 65534, 32767, 41, 1] avg=32768.0000 max=65534 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32768, 32426, 33096, 33096, 32767, 32456, 32439] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=95468 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=43 tokens_per_ep_rank=[67779, 2, 7447, 4, 32770, 55843, 65531, 32768] avg=32768.0000 max=67779 max_rank=0 min=2 min_rank=1 imbalance_ratio=2.068451 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=43 tokens_per_ep_rank=[33096, 32767, 31455, 33096, 32770, 33096, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31455 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34683 max_load_reduction_pct=0.511707 moved_to_replicas=89865 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=44 tokens_per_ep_rank=[32768, 4, 53475, 73843, 3826, 32770, 65451, 7] avg=32768.0000 max=73843 max_rank=3 min=4 min_rank=1 imbalance_ratio=2.253510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=44 tokens_per_ep_rank=[32768, 32769, 33096, 33096, 33096, 32770, 33096, 31453] avg=32768.0000 max=33096 max_rank=2 min=31453 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=40747 max_load_reduction_pct=0.551806 moved_to_replicas=93481 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=45 tokens_per_ep_rank=[32739, 57638, 32768, 603, 32906, 40014, 32704, 32772] avg=32768.0000 max=57638 max_rank=1 min=603 min_rank=3 imbalance_ratio=1.758972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=45 tokens_per_ep_rank=[32739, 33096, 32768, 32061, 32906, 33098, 32704, 32772] avg=32768.0000 max=33098 max_rank=5 min=32061 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=24540 max_load_reduction_pct=0.425761 moved_to_replicas=31460 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=46 tokens_per_ep_rank=[32762, 4, 4, 32766, 32767, 28, 131121, 32692] avg=32768.0000 max=131121 max_rank=6 min=4 min_rank=1 imbalance_ratio=4.001495 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=46 tokens_per_ep_rank=[32762, 32769, 32769, 32766, 32767, 32523, 33096, 32692] avg=32768.0000 max=33096 max_rank=6 min=32523 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98025 max_load_reduction_pct=0.747592 moved_to_replicas=98025 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=47 tokens_per_ep_rank=[5, 32761, 32768, 65605, 32766, 33147, 65092, 0] avg=32768.0000 max=65605 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.002106 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=47 tokens_per_ep_rank=[32052, 32761, 32768, 33096, 32766, 33096, 33096, 32509] avg=32768.0000 max=33096 max_rank=3 min=32052 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64556 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 65534, 2, 4, 47173, 30881, 53071] avg=32768.0000 max=65534 max_rank=2 min=2 min_rank=3 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=48 tokens_per_ep_rank=[32715, 32764, 33096, 32440, 33098, 33095, 31841, 33095] avg=32768.0000 max=33098 max_rank=4 min=31841 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32436 max_load_reduction_pct=0.494949 moved_to_replicas=66490 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=48 total_expected_recv=1573034 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376192 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=48 total_expected_recv=1574327 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374400 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=48 total_expected_recv=1569897 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379264 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=48 total_expected_recv=1574031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375296 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=48 total_expected_recv=1568466 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380928 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=48 total_expected_recv=1574590 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375680 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=48 total_expected_recv=1576372 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372096 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=48 total_expected_recv=1572195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379136 peak_demand=33096 required_capacity=1.010010 - [2026-08-26 15:01:26] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 730.4 | throughput per GPU (TFLOP/s/GPU): 122.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.781345E+00 | loss scale: 1.0 | grad norm: 2.914 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32264 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=31407 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32264 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=31407 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=31151 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=31151 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32415 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32415 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31752 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31752 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31815 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32002 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[23213, 26981, 34503, 46652, 41308, 23620, 27365, 38502] avg=32768.0000 max=46652 max_rank=3 min=23213 min_rank=0 imbalance_ratio=1.423706 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32002 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[23213, 26981, 34503, 46652, 41308, 23620, 27365, 38502] avg=32768.0000 max=46652 max_rank=3 min=23213 min_rank=0 imbalance_ratio=1.423706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[23213, 26981, 34503, 46652, 41308, 23620, 27365, 38502] avg=32768.0000 max=46652 max_rank=3 min=23213 min_rank=0 imbalance_ratio=1.423706 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] avg=32768.0000 max=33097 max_rank=3 min=31094 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=13555 max_load_reduction_pct=0.290556 moved_to_replicas=28581 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] avg=32768.0000 max=33097 max_rank=3 min=31094 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=13555 max_load_reduction_pct=0.290556 moved_to_replicas=28581 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] avg=32768.0000 max=33097 max_rank=3 min=31094 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=13555 max_load_reduction_pct=0.290556 moved_to_replicas=28581 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] avg=32768.0000 max=33097 max_rank=3 min=31094 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=13555 max_load_reduction_pct=0.290556 moved_to_replicas=28581 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[77173, 199, 4598, 49165, 19506, 50693, 31709, 29101] avg=32768.0000 max=77173 max_rank=0 min=199 min_rank=1 imbalance_ratio=2.355133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[23213, 26981, 34503, 46652, 41308, 23620, 27365, 38502] avg=32768.0000 max=46652 max_rank=3 min=23213 min_rank=0 imbalance_ratio=1.423706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[23213, 26981, 34503, 46652, 41308, 23620, 27365, 38502] avg=32768.0000 max=46652 max_rank=3 min=23213 min_rank=0 imbalance_ratio=1.423706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[77173, 199, 4598, 49165, 19506, 50693, 31709, 29101] avg=32768.0000 max=77173 max_rank=0 min=199 min_rank=1 imbalance_ratio=2.355133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[77173, 199, 4598, 49165, 19506, 50693, 31709, 29101] avg=32768.0000 max=77173 max_rank=0 min=199 min_rank=1 imbalance_ratio=2.355133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] avg=32768.0000 max=33097 max_rank=4 min=31709 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=44076 max_load_reduction_pct=0.571132 moved_to_replicas=77743 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[77173, 199, 4598, 49165, 19506, 50693, 31709, 29101] avg=32768.0000 max=77173 max_rank=0 min=199 min_rank=1 imbalance_ratio=2.355133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] avg=32768.0000 max=33097 max_rank=3 min=31094 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=13555 max_load_reduction_pct=0.290556 moved_to_replicas=28581 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] avg=32768.0000 max=33097 max_rank=3 min=31094 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=13555 max_load_reduction_pct=0.290556 moved_to_replicas=28581 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[23213, 26981, 34503, 46652, 41308, 23620, 27365, 38502] avg=32768.0000 max=46652 max_rank=3 min=23213 min_rank=0 imbalance_ratio=1.423706 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] avg=32768.0000 max=33097 max_rank=4 min=31709 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=44076 max_load_reduction_pct=0.571132 moved_to_replicas=77743 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] avg=32768.0000 max=33097 max_rank=4 min=31709 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=44076 max_load_reduction_pct=0.571132 moved_to_replicas=77743 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] avg=32768.0000 max=33097 max_rank=4 min=31709 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=44076 max_load_reduction_pct=0.571132 moved_to_replicas=77743 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[87509, 10579, 14782, 6530, 19366, 71062, 21063, 31253] avg=32768.0000 max=87509 max_rank=0 min=6530 min_rank=3 imbalance_ratio=2.670563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] avg=32768.0000 max=33097 max_rank=3 min=31094 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=13555 max_load_reduction_pct=0.290556 moved_to_replicas=28581 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[77173, 199, 4598, 49165, 19506, 50693, 31709, 29101] avg=32768.0000 max=77173 max_rank=0 min=199 min_rank=1 imbalance_ratio=2.355133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[77173, 199, 4598, 49165, 19506, 50693, 31709, 29101] avg=32768.0000 max=77173 max_rank=0 min=199 min_rank=1 imbalance_ratio=2.355133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[87509, 10579, 14782, 6530, 19366, 71062, 21063, 31253] avg=32768.0000 max=87509 max_rank=0 min=6530 min_rank=3 imbalance_ratio=2.670563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[87509, 10579, 14782, 6530, 19366, 71062, 21063, 31253] avg=32768.0000 max=87509 max_rank=0 min=6530 min_rank=3 imbalance_ratio=2.670563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] avg=32768.0000 max=33098 max_rank=5 min=31253 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=54411 max_load_reduction_pct=0.621776 moved_to_replicas=92379 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[87509, 10579, 14782, 6530, 19366, 71062, 21063, 31253] avg=32768.0000 max=87509 max_rank=0 min=6530 min_rank=3 imbalance_ratio=2.670563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[23213, 26981, 34503, 46652, 41308, 23620, 27365, 38502] avg=32768.0000 max=46652 max_rank=3 min=23213 min_rank=0 imbalance_ratio=1.423706 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] avg=32768.0000 max=33097 max_rank=4 min=31709 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=44076 max_load_reduction_pct=0.571132 moved_to_replicas=77743 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] avg=32768.0000 max=33097 max_rank=4 min=31709 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=44076 max_load_reduction_pct=0.571132 moved_to_replicas=77743 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] avg=32768.0000 max=33098 max_rank=5 min=31253 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=54411 max_load_reduction_pct=0.621776 moved_to_replicas=92379 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[77173, 199, 4598, 49165, 19506, 50693, 31709, 29101] avg=32768.0000 max=77173 max_rank=0 min=199 min_rank=1 imbalance_ratio=2.355133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] avg=32768.0000 max=33098 max_rank=5 min=31253 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=54411 max_load_reduction_pct=0.621776 moved_to_replicas=92379 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] avg=32768.0000 max=33098 max_rank=5 min=31253 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=54411 max_load_reduction_pct=0.621776 moved_to_replicas=92379 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[503, 11, 48995, 18604, 20757, 73337, 63575, 36362] avg=32768.0000 max=73337 max_rank=5 min=11 min_rank=1 imbalance_ratio=2.238068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32475, 33096, 33095, 33097, 33096, 33095, 31094, 33096] avg=32768.0000 max=33097 max_rank=3 min=31094 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=13555 max_load_reduction_pct=0.290556 moved_to_replicas=28581 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] avg=32768.0000 max=33097 max_rank=4 min=31709 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=44076 max_load_reduction_pct=0.571132 moved_to_replicas=77743 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[87509, 10579, 14782, 6530, 19366, 71062, 21063, 31253] avg=32768.0000 max=87509 max_rank=0 min=6530 min_rank=3 imbalance_ratio=2.670563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[87509, 10579, 14782, 6530, 19366, 71062, 21063, 31253] avg=32768.0000 max=87509 max_rank=0 min=6530 min_rank=3 imbalance_ratio=2.670563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[503, 11, 48995, 18604, 20757, 73337, 63575, 36362] avg=32768.0000 max=73337 max_rank=5 min=11 min_rank=1 imbalance_ratio=2.238068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[503, 11, 48995, 18604, 20757, 73337, 63575, 36362] avg=32768.0000 max=73337 max_rank=5 min=11 min_rank=1 imbalance_ratio=2.238068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=31775 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40241 max_load_reduction_pct=0.548713 moved_to_replicas=89885 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[503, 11, 48995, 18604, 20757, 73337, 63575, 36362] avg=32768.0000 max=73337 max_rank=5 min=11 min_rank=1 imbalance_ratio=2.238068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] avg=32768.0000 max=33098 max_rank=5 min=31253 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=54411 max_load_reduction_pct=0.621776 moved_to_replicas=92379 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] avg=32768.0000 max=33098 max_rank=5 min=31253 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=54411 max_load_reduction_pct=0.621776 moved_to_replicas=92379 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=31775 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40241 max_load_reduction_pct=0.548713 moved_to_replicas=89885 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[77173, 199, 4598, 49165, 19506, 50693, 31709, 29101] avg=32768.0000 max=77173 max_rank=0 min=199 min_rank=1 imbalance_ratio=2.355133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=31775 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40241 max_load_reduction_pct=0.548713 moved_to_replicas=89885 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[87509, 10579, 14782, 6530, 19366, 71062, 21063, 31253] avg=32768.0000 max=87509 max_rank=0 min=6530 min_rank=3 imbalance_ratio=2.670563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=31775 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40241 max_load_reduction_pct=0.548713 moved_to_replicas=89885 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[51375, 32778, 45629, 52867, 14029, 19902, 2, 45562] avg=32768.0000 max=52867 max_rank=3 min=2 min_rank=6 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 33097, 33096, 31709, 31859] avg=32768.0000 max=33097 max_rank=4 min=31709 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=44076 max_load_reduction_pct=0.571132 moved_to_replicas=77743 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] avg=32768.0000 max=33098 max_rank=5 min=31253 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=54411 max_load_reduction_pct=0.621776 moved_to_replicas=92379 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[503, 11, 48995, 18604, 20757, 73337, 63575, 36362] avg=32768.0000 max=73337 max_rank=5 min=11 min_rank=1 imbalance_ratio=2.238068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[503, 11, 48995, 18604, 20757, 73337, 63575, 36362] avg=32768.0000 max=73337 max_rank=5 min=11 min_rank=1 imbalance_ratio=2.238068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[51375, 32778, 45629, 52867, 14029, 19902, 2, 45562] avg=32768.0000 max=52867 max_rank=3 min=2 min_rank=6 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[51375, 32778, 45629, 52867, 14029, 19902, 2, 45562] avg=32768.0000 max=52867 max_rank=3 min=2 min_rank=6 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] avg=32768.0000 max=33097 max_rank=0 min=32304 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=63049 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[51375, 32778, 45629, 52867, 14029, 19902, 2, 45562] avg=32768.0000 max=52867 max_rank=3 min=2 min_rank=6 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=31775 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40241 max_load_reduction_pct=0.548713 moved_to_replicas=89885 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=31775 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40241 max_load_reduction_pct=0.548713 moved_to_replicas=89885 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] avg=32768.0000 max=33097 max_rank=0 min=32304 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=63049 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] avg=32768.0000 max=33097 max_rank=0 min=32304 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=63049 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[87509, 10579, 14782, 6530, 19366, 71062, 21063, 31253] avg=32768.0000 max=87509 max_rank=0 min=6530 min_rank=3 imbalance_ratio=2.670563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[503, 11, 48995, 18604, 20757, 73337, 63575, 36362] avg=32768.0000 max=73337 max_rank=5 min=11 min_rank=1 imbalance_ratio=2.238068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] avg=32768.0000 max=33097 max_rank=0 min=32304 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=63049 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[83965, 4091, 47215, 3841, 26089, 127, 31169, 65647] avg=32768.0000 max=83965 max_rank=0 min=127 min_rank=5 imbalance_ratio=2.562408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[51375, 32778, 45629, 52867, 14029, 19902, 2, 45562] avg=32768.0000 max=52867 max_rank=3 min=2 min_rank=6 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[83965, 4091, 47215, 3841, 26089, 127, 31169, 65647] avg=32768.0000 max=83965 max_rank=0 min=127 min_rank=5 imbalance_ratio=2.562408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[51375, 32778, 45629, 52867, 14029, 19902, 2, 45562] avg=32768.0000 max=52867 max_rank=3 min=2 min_rank=6 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=31775 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40241 max_load_reduction_pct=0.548713 moved_to_replicas=89885 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[33095, 33095, 33097, 33096, 33096, 33098, 32314, 31253] avg=32768.0000 max=33098 max_rank=5 min=31253 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=54411 max_load_reduction_pct=0.621776 moved_to_replicas=92379 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[83965, 4091, 47215, 3841, 26089, 127, 31169, 65647] avg=32768.0000 max=83965 max_rank=0 min=127 min_rank=5 imbalance_ratio=2.562408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50869 max_load_reduction_pct=0.605836 moved_to_replicas=97539 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[83965, 4091, 47215, 3841, 26089, 127, 31169, 65647] avg=32768.0000 max=83965 max_rank=0 min=127 min_rank=5 imbalance_ratio=2.562408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] avg=32768.0000 max=33097 max_rank=0 min=32304 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=63049 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50869 max_load_reduction_pct=0.605836 moved_to_replicas=97539 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] avg=32768.0000 max=33097 max_rank=0 min=32304 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=63049 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50869 max_load_reduction_pct=0.605836 moved_to_replicas=97539 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50869 max_load_reduction_pct=0.605836 moved_to_replicas=97539 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[51375, 32778, 45629, 52867, 14029, 19902, 2, 45562] avg=32768.0000 max=52867 max_rank=3 min=2 min_rank=6 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[503, 11, 48995, 18604, 20757, 73337, 63575, 36362] avg=32768.0000 max=73337 max_rank=5 min=11 min_rank=1 imbalance_ratio=2.238068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[4, 65562, 65320, 2532, 32785, 32784, 24863, 38294] avg=32768.0000 max=65562 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[83965, 4091, 47215, 3841, 26089, 127, 31169, 65647] avg=32768.0000 max=83965 max_rank=0 min=127 min_rank=5 imbalance_ratio=2.562408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[4, 65562, 65320, 2532, 32785, 32784, 24863, 38294] avg=32768.0000 max=65562 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[83965, 4091, 47215, 3841, 26089, 127, 31169, 65647] avg=32768.0000 max=83965 max_rank=0 min=127 min_rank=5 imbalance_ratio=2.562408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[4, 65562, 65320, 2532, 32785, 32784, 24863, 38294] avg=32768.0000 max=65562 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] avg=32768.0000 max=33097 max_rank=0 min=32304 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=63049 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] avg=32768.0000 max=33096 max_rank=1 min=31721 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32466 max_load_reduction_pct=0.495195 moved_to_replicas=69888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[33095, 31775, 33096, 31795, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=2 min=31775 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40241 max_load_reduction_pct=0.548713 moved_to_replicas=89885 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[4, 65562, 65320, 2532, 32785, 32784, 24863, 38294] avg=32768.0000 max=65562 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50869 max_load_reduction_pct=0.605836 moved_to_replicas=97539 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] avg=32768.0000 max=33096 max_rank=1 min=31721 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32466 max_load_reduction_pct=0.495195 moved_to_replicas=69888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50869 max_load_reduction_pct=0.605836 moved_to_replicas=97539 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] avg=32768.0000 max=33096 max_rank=1 min=31721 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32466 max_load_reduction_pct=0.495195 moved_to_replicas=69888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] avg=32768.0000 max=33096 max_rank=1 min=31721 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32466 max_load_reduction_pct=0.495195 moved_to_replicas=69888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[83965, 4091, 47215, 3841, 26089, 127, 31169, 65647] avg=32768.0000 max=83965 max_rank=0 min=127 min_rank=5 imbalance_ratio=2.562408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[94005, 2008, 71520, 2, 32813, 32912, 28756, 128] avg=32768.0000 max=94005 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.868805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[4, 65562, 65320, 2532, 32785, 32784, 24863, 38294] avg=32768.0000 max=65562 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[51375, 32778, 45629, 52867, 14029, 19902, 2, 45562] avg=32768.0000 max=52867 max_rank=3 min=2 min_rank=6 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[94005, 2008, 71520, 2, 32813, 32912, 28756, 128] avg=32768.0000 max=94005 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.868805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[4, 65562, 65320, 2532, 32785, 32784, 24863, 38294] avg=32768.0000 max=65562 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[94005, 2008, 71520, 2, 32813, 32912, 28756, 128] avg=32768.0000 max=94005 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.868805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50869 max_load_reduction_pct=0.605836 moved_to_replicas=97539 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] avg=32768.0000 max=33097 max_rank=2 min=31269 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=60908 max_load_reduction_pct=0.647923 moved_to_replicas=99333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[94005, 2008, 71520, 2, 32813, 32912, 28756, 128] avg=32768.0000 max=94005 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.868805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] avg=32768.0000 max=33096 max_rank=1 min=31721 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32466 max_load_reduction_pct=0.495195 moved_to_replicas=69888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[33097, 32778, 33097, 33097, 32307, 32369, 32304, 33095] avg=32768.0000 max=33097 max_rank=0 min=32304 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=63049 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] avg=32768.0000 max=33097 max_rank=2 min=31269 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=60908 max_load_reduction_pct=0.647923 moved_to_replicas=99333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] avg=32768.0000 max=33096 max_rank=1 min=31721 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32466 max_load_reduction_pct=0.495195 moved_to_replicas=69888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] avg=32768.0000 max=33097 max_rank=2 min=31269 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=60908 max_load_reduction_pct=0.647923 moved_to_replicas=99333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] avg=32768.0000 max=33097 max_rank=2 min=31269 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=60908 max_load_reduction_pct=0.647923 moved_to_replicas=99333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65574, 69600, 2156, 87250, 4, 1128, 3657, 32775] avg=32768.0000 max=87250 max_rank=3 min=4 min_rank=4 imbalance_ratio=2.662659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[4, 65562, 65320, 2532, 32785, 32784, 24863, 38294] avg=32768.0000 max=65562 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[94005, 2008, 71520, 2, 32813, 32912, 28756, 128] avg=32768.0000 max=94005 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.868805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65574, 69600, 2156, 87250, 4, 1128, 3657, 32775] avg=32768.0000 max=87250 max_rank=3 min=4 min_rank=4 imbalance_ratio=2.662659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[83965, 4091, 47215, 3841, 26089, 127, 31169, 65647] avg=32768.0000 max=83965 max_rank=0 min=127 min_rank=5 imbalance_ratio=2.562408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[94005, 2008, 71520, 2, 32813, 32912, 28756, 128] avg=32768.0000 max=94005 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.868805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65574, 69600, 2156, 87250, 4, 1128, 3657, 32775] avg=32768.0000 max=87250 max_rank=3 min=4 min_rank=4 imbalance_ratio=2.662659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] avg=32768.0000 max=33097 max_rank=3 min=31478 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54153 max_load_reduction_pct=0.620665 moved_to_replicas=123136 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65574, 69600, 2156, 87250, 4, 1128, 3657, 32775] avg=32768.0000 max=87250 max_rank=3 min=4 min_rank=4 imbalance_ratio=2.662659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] avg=32768.0000 max=33096 max_rank=1 min=31721 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32466 max_load_reduction_pct=0.495195 moved_to_replicas=69888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] avg=32768.0000 max=33097 max_rank=2 min=31269 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=60908 max_load_reduction_pct=0.647923 moved_to_replicas=99333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] avg=32768.0000 max=33097 max_rank=3 min=31478 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54153 max_load_reduction_pct=0.620665 moved_to_replicas=123136 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32605, 32890, 31169, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50869 max_load_reduction_pct=0.605836 moved_to_replicas=97539 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] avg=32768.0000 max=33097 max_rank=3 min=31478 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54153 max_load_reduction_pct=0.620665 moved_to_replicas=123136 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] avg=32768.0000 max=33097 max_rank=2 min=31269 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=60908 max_load_reduction_pct=0.647923 moved_to_replicas=99333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] avg=32768.0000 max=33097 max_rank=3 min=31478 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54153 max_load_reduction_pct=0.620665 moved_to_replicas=123136 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[0, 32759, 64717, 15833, 4, 48267, 32575, 67989] avg=32768.0000 max=67989 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.074860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[0, 32759, 64717, 15833, 4, 48267, 32575, 67989] avg=32768.0000 max=67989 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.074860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65574, 69600, 2156, 87250, 4, 1128, 3657, 32775] avg=32768.0000 max=87250 max_rank=3 min=4 min_rank=4 imbalance_ratio=2.662659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[94005, 2008, 71520, 2, 32813, 32912, 28756, 128] avg=32768.0000 max=94005 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.868805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[4, 65562, 65320, 2532, 32785, 32784, 24863, 38294] avg=32768.0000 max=65562 max_rank=1 min=4 min_rank=0 imbalance_ratio=2.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] avg=32768.0000 max=33096 max_rank=2 min=31661 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34893 max_load_reduction_pct=0.513215 moved_to_replicas=81685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[0, 32759, 64717, 15833, 4, 48267, 32575, 67989] avg=32768.0000 max=67989 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.074860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] avg=32768.0000 max=33096 max_rank=2 min=31661 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34893 max_load_reduction_pct=0.513215 moved_to_replicas=81685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65574, 69600, 2156, 87250, 4, 1128, 3657, 32775] avg=32768.0000 max=87250 max_rank=3 min=4 min_rank=4 imbalance_ratio=2.662659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] avg=32768.0000 max=33097 max_rank=3 min=31478 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54153 max_load_reduction_pct=0.620665 moved_to_replicas=123136 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[0, 32759, 64717, 15833, 4, 48267, 32575, 67989] avg=32768.0000 max=67989 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.074860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32470, 33096, 33096, 33096, 32785, 32784, 31721, 33096] avg=32768.0000 max=33096 max_rank=1 min=31721 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32466 max_load_reduction_pct=0.495195 moved_to_replicas=69888 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] avg=32768.0000 max=33097 max_rank=2 min=31269 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=60908 max_load_reduction_pct=0.647923 moved_to_replicas=99333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] avg=32768.0000 max=33096 max_rank=2 min=31661 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34893 max_load_reduction_pct=0.513215 moved_to_replicas=81685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[2, 3877, 89608, 73037, 32769, 32768, 30076, 7] avg=32768.0000 max=89608 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.734619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] avg=32768.0000 max=33097 max_rank=3 min=31478 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54153 max_load_reduction_pct=0.620665 moved_to_replicas=123136 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] avg=32768.0000 max=33096 max_rank=2 min=31661 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34893 max_load_reduction_pct=0.513215 moved_to_replicas=81685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[2, 3877, 89608, 73037, 32769, 32768, 30076, 7] avg=32768.0000 max=89608 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.734619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[0, 32759, 64717, 15833, 4, 48267, 32575, 67989] avg=32768.0000 max=67989 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.074860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65574, 69600, 2156, 87250, 4, 1128, 3657, 32775] avg=32768.0000 max=87250 max_rank=3 min=4 min_rank=4 imbalance_ratio=2.662659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[94005, 2008, 71520, 2, 32813, 32912, 28756, 128] avg=32768.0000 max=94005 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.868805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] avg=32768.0000 max=33097 max_rank=7 min=31456 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=56511 max_load_reduction_pct=0.630647 moved_to_replicas=96453 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] avg=32768.0000 max=33097 max_rank=7 min=31456 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=56511 max_load_reduction_pct=0.630647 moved_to_replicas=96453 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[2, 3877, 89608, 73037, 32769, 32768, 30076, 7] avg=32768.0000 max=89608 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.734619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] avg=32768.0000 max=33096 max_rank=2 min=31661 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34893 max_load_reduction_pct=0.513215 moved_to_replicas=81685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[0, 32759, 64717, 15833, 4, 48267, 32575, 67989] avg=32768.0000 max=67989 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.074860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] avg=32768.0000 max=33097 max_rank=3 min=31478 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54153 max_load_reduction_pct=0.620665 moved_to_replicas=123136 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[2, 3877, 89608, 73037, 32769, 32768, 30076, 7] avg=32768.0000 max=89608 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.734619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33096, 33096, 33097, 32766, 32813, 32912, 33095, 31269] avg=32768.0000 max=33097 max_rank=2 min=31269 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=60908 max_load_reduction_pct=0.647923 moved_to_replicas=99333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] avg=32768.0000 max=33097 max_rank=7 min=31456 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=56511 max_load_reduction_pct=0.630647 moved_to_replicas=96453 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 62134, 36004, 12] avg=32768.0000 max=62134 max_rank=5 min=12 min_rank=7 imbalance_ratio=1.896179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] avg=32768.0000 max=33096 max_rank=2 min=31661 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34893 max_load_reduction_pct=0.513215 moved_to_replicas=81685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 62134, 36004, 12] avg=32768.0000 max=62134 max_rank=5 min=12 min_rank=7 imbalance_ratio=1.896179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] avg=32768.0000 max=33097 max_rank=7 min=31456 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=56511 max_load_reduction_pct=0.630647 moved_to_replicas=96453 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[2, 3877, 89608, 73037, 32769, 32768, 30076, 7] avg=32768.0000 max=89608 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.734619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[0, 32759, 64717, 15833, 4, 48267, 32575, 67989] avg=32768.0000 max=67989 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.074860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65574, 69600, 2156, 87250, 4, 1128, 3657, 32775] avg=32768.0000 max=87250 max_rank=3 min=4 min_rank=4 imbalance_ratio=2.662659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] avg=32768.0000 max=33096 max_rank=5 min=31958 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29038 max_load_reduction_pct=0.467345 moved_to_replicas=31946 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] avg=32768.0000 max=33096 max_rank=5 min=31958 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29038 max_load_reduction_pct=0.467345 moved_to_replicas=31946 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 62134, 36004, 12] avg=32768.0000 max=62134 max_rank=5 min=12 min_rank=7 imbalance_ratio=1.896179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] avg=32768.0000 max=33097 max_rank=7 min=31456 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=56511 max_load_reduction_pct=0.630647 moved_to_replicas=96453 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[2, 3877, 89608, 73037, 32769, 32768, 30076, 7] avg=32768.0000 max=89608 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.734619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 62134, 36004, 12] avg=32768.0000 max=62134 max_rank=5 min=12 min_rank=7 imbalance_ratio=1.896179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] avg=32768.0000 max=33096 max_rank=2 min=31661 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34893 max_load_reduction_pct=0.513215 moved_to_replicas=81685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 31478, 32410, 33096, 32775] avg=32768.0000 max=33097 max_rank=3 min=31478 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=54153 max_load_reduction_pct=0.620665 moved_to_replicas=123136 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] avg=32768.0000 max=33096 max_rank=5 min=31958 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29038 max_load_reduction_pct=0.467345 moved_to_replicas=31946 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32768, 17977, 50243, 32766, 16, 95597, 9, 32768] avg=32768.0000 max=95597 max_rank=5 min=9 min_rank=6 imbalance_ratio=2.917389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32768, 17977, 50243, 32766, 16, 95597, 9, 32768] avg=32768.0000 max=95597 max_rank=5 min=9 min_rank=6 imbalance_ratio=2.917389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] avg=32768.0000 max=33097 max_rank=7 min=31456 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=56511 max_load_reduction_pct=0.630647 moved_to_replicas=96453 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] avg=32768.0000 max=33096 max_rank=5 min=31958 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29038 max_load_reduction_pct=0.467345 moved_to_replicas=31946 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 62134, 36004, 12] avg=32768.0000 max=62134 max_rank=5 min=12 min_rank=7 imbalance_ratio=1.896179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=62499 max_load_reduction_pct=0.653776 moved_to_replicas=79648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[0, 32759, 64717, 15833, 4, 48267, 32575, 67989] avg=32768.0000 max=67989 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.074860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[2, 3877, 89608, 73037, 32769, 32768, 30076, 7] avg=32768.0000 max=89608 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.734619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=62499 max_load_reduction_pct=0.653776 moved_to_replicas=79648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32768, 17977, 50243, 32766, 16, 95597, 9, 32768] avg=32768.0000 max=95597 max_rank=5 min=9 min_rank=6 imbalance_ratio=2.917389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] avg=32768.0000 max=33096 max_rank=5 min=31958 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29038 max_load_reduction_pct=0.467345 moved_to_replicas=31946 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 62134, 36004, 12] avg=32768.0000 max=62134 max_rank=5 min=12 min_rank=7 imbalance_ratio=1.896179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32768, 17977, 50243, 32766, 16, 95597, 9, 32768] avg=32768.0000 max=95597 max_rank=5 min=9 min_rank=6 imbalance_ratio=2.917389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] avg=32768.0000 max=33097 max_rank=7 min=31456 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=56511 max_load_reduction_pct=0.630647 moved_to_replicas=96453 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32765, 32759, 33096, 31661, 33096, 33096, 32575, 33096] avg=32768.0000 max=33096 max_rank=2 min=31661 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34893 max_load_reduction_pct=0.513215 moved_to_replicas=81685 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=62499 max_load_reduction_pct=0.653776 moved_to_replicas=79648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[88182, 32765, 9594, 32768, 32762, 32740, 522, 32811] avg=32768.0000 max=88182 max_rank=0 min=522 min_rank=6 imbalance_ratio=2.691101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[88182, 32765, 9594, 32768, 32762, 32740, 522, 32811] avg=32768.0000 max=88182 max_rank=0 min=522 min_rank=6 imbalance_ratio=2.691101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=62499 max_load_reduction_pct=0.653776 moved_to_replicas=79648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32768, 17977, 50243, 32766, 16, 95597, 9, 32768] avg=32768.0000 max=95597 max_rank=5 min=9 min_rank=6 imbalance_ratio=2.917389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] avg=32768.0000 max=33096 max_rank=5 min=31958 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29038 max_load_reduction_pct=0.467345 moved_to_replicas=31946 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] avg=32768.0000 max=33097 max_rank=0 min=32105 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55085 max_load_reduction_pct=0.624674 moved_to_replicas=55086 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 62134, 36004, 12] avg=32768.0000 max=62134 max_rank=5 min=12 min_rank=7 imbalance_ratio=1.896179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] avg=32768.0000 max=33097 max_rank=0 min=32105 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55085 max_load_reduction_pct=0.624674 moved_to_replicas=55086 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[2, 3877, 89608, 73037, 32769, 32768, 30076, 7] avg=32768.0000 max=89608 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.734619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[88182, 32765, 9594, 32768, 32762, 32740, 522, 32811] avg=32768.0000 max=88182 max_rank=0 min=522 min_rank=6 imbalance_ratio=2.691101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=62499 max_load_reduction_pct=0.653776 moved_to_replicas=79648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[88182, 32765, 9594, 32768, 32762, 32740, 522, 32811] avg=32768.0000 max=88182 max_rank=0 min=522 min_rank=6 imbalance_ratio=2.691101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32768, 17977, 50243, 32766, 16, 95597, 9, 32768] avg=32768.0000 max=95597 max_rank=5 min=9 min_rank=6 imbalance_ratio=2.917389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] avg=32768.0000 max=33096 max_rank=5 min=31958 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29038 max_load_reduction_pct=0.467345 moved_to_replicas=31946 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[32767, 33096, 33095, 33096, 32769, 32768, 31456, 33097] avg=32768.0000 max=33097 max_rank=7 min=31456 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=56511 max_load_reduction_pct=0.630647 moved_to_replicas=96453 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] avg=32768.0000 max=33097 max_rank=0 min=32105 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55085 max_load_reduction_pct=0.624674 moved_to_replicas=55086 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32802, 135387, 32738, 6, 3129, 47089, 10897, 96] avg=32768.0000 max=135387 max_rank=1 min=6 min_rank=3 imbalance_ratio=4.131683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32802, 135387, 32738, 6, 3129, 47089, 10897, 96] avg=32768.0000 max=135387 max_rank=1 min=6 min_rank=3 imbalance_ratio=4.131683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] avg=32768.0000 max=33097 max_rank=0 min=32105 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55085 max_load_reduction_pct=0.624674 moved_to_replicas=55086 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[88182, 32765, 9594, 32768, 32762, 32740, 522, 32811] avg=32768.0000 max=88182 max_rank=0 min=522 min_rank=6 imbalance_ratio=2.691101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=62499 max_load_reduction_pct=0.653776 moved_to_replicas=79648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] avg=32768.0000 max=33096 max_rank=1 min=31684 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102291 max_load_reduction_pct=0.755545 moved_to_replicas=116284 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] avg=32768.0000 max=33096 max_rank=1 min=31684 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102291 max_load_reduction_pct=0.755545 moved_to_replicas=116284 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32768, 17977, 50243, 32766, 16, 95597, 9, 32768] avg=32768.0000 max=95597 max_rank=5 min=9 min_rank=6 imbalance_ratio=2.917389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 62134, 36004, 12] avg=32768.0000 max=62134 max_rank=5 min=12 min_rank=7 imbalance_ratio=1.896179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32802, 135387, 32738, 6, 3129, 47089, 10897, 96] avg=32768.0000 max=135387 max_rank=1 min=6 min_rank=3 imbalance_ratio=4.131683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] avg=32768.0000 max=33097 max_rank=0 min=32105 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55085 max_load_reduction_pct=0.624674 moved_to_replicas=55086 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32802, 135387, 32738, 6, 3129, 47089, 10897, 96] avg=32768.0000 max=135387 max_rank=1 min=6 min_rank=3 imbalance_ratio=4.131683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[88182, 32765, 9594, 32768, 32762, 32740, 522, 32811] avg=32768.0000 max=88182 max_rank=0 min=522 min_rank=6 imbalance_ratio=2.691101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=62499 max_load_reduction_pct=0.653776 moved_to_replicas=79648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32951, 32743, 32773, 32713, 32814, 33096, 33096, 31958] avg=32768.0000 max=33096 max_rank=5 min=31958 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=29038 max_load_reduction_pct=0.467345 moved_to_replicas=31946 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[32550, 0, 65497, 36177, 29536, 65539, 32736, 109] avg=32768.0000 max=65539 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] avg=32768.0000 max=33096 max_rank=1 min=31684 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102291 max_load_reduction_pct=0.755545 moved_to_replicas=116284 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[32550, 0, 65497, 36177, 29536, 65539, 32736, 109] avg=32768.0000 max=65539 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] avg=32768.0000 max=33096 max_rank=1 min=31684 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102291 max_load_reduction_pct=0.755545 moved_to_replicas=116284 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32802, 135387, 32738, 6, 3129, 47089, 10897, 96] avg=32768.0000 max=135387 max_rank=1 min=6 min_rank=3 imbalance_ratio=4.131683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] avg=32768.0000 max=33097 max_rank=0 min=32105 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55085 max_load_reduction_pct=0.624674 moved_to_replicas=55086 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=67925 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=67925 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[88182, 32765, 9594, 32768, 32762, 32740, 522, 32811] avg=32768.0000 max=88182 max_rank=0 min=522 min_rank=6 imbalance_ratio=2.691101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32768, 17977, 50243, 32766, 16, 95597, 9, 32768] avg=32768.0000 max=95597 max_rank=5 min=9 min_rank=6 imbalance_ratio=2.917389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[32550, 0, 65497, 36177, 29536, 65539, 32736, 109] avg=32768.0000 max=65539 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] avg=32768.0000 max=33096 max_rank=1 min=31684 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102291 max_load_reduction_pct=0.755545 moved_to_replicas=116284 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[32550, 0, 65497, 36177, 29536, 65539, 32736, 109] avg=32768.0000 max=65539 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32802, 135387, 32738, 6, 3129, 47089, 10897, 96] avg=32768.0000 max=135387 max_rank=1 min=6 min_rank=3 imbalance_ratio=4.131683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] avg=32768.0000 max=33097 max_rank=0 min=32105 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55085 max_load_reduction_pct=0.624674 moved_to_replicas=55086 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 19, 65525, 0, 32808, 65524, 32768] avg=32768.0000 max=65525 max_rank=3 min=0 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=67925 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32768, 33094, 33098, 32766, 31783, 33096, 32771, 32768] avg=32768.0000 max=33098 max_rank=2 min=31783 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=62499 max_load_reduction_pct=0.653776 moved_to_replicas=79648 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 19, 65525, 0, 32808, 65524, 32768] avg=32768.0000 max=65525 max_rank=3 min=0 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=67925 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[32550, 0, 65497, 36177, 29536, 65539, 32736, 109] avg=32768.0000 max=65539 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] avg=32768.0000 max=33096 max_rank=1 min=31684 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102291 max_load_reduction_pct=0.755545 moved_to_replicas=116284 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=32429 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=64857 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=32429 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=64857 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32802, 135387, 32738, 6, 3129, 47089, 10897, 96] avg=32768.0000 max=135387 max_rank=1 min=6 min_rank=3 imbalance_ratio=4.131683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 19, 65525, 0, 32808, 65524, 32768] avg=32768.0000 max=65525 max_rank=3 min=0 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=67925 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[88182, 32765, 9594, 32768, 32762, 32740, 522, 32811] avg=32768.0000 max=88182 max_rank=0 min=522 min_rank=6 imbalance_ratio=2.691101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 19, 65525, 0, 32808, 65524, 32768] avg=32768.0000 max=65525 max_rank=3 min=0 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[32550, 0, 65497, 36177, 29536, 65539, 32736, 109] avg=32768.0000 max=65539 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 65557, 4, 66195, 32766, 2] avg=32768.0000 max=66195 max_rank=5 min=2 min_rank=7 imbalance_ratio=2.020111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] avg=32768.0000 max=33096 max_rank=1 min=31684 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102291 max_load_reduction_pct=0.755545 moved_to_replicas=116284 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=32429 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=64857 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 65557, 4, 66195, 32766, 2] avg=32768.0000 max=66195 max_rank=5 min=2 min_rank=7 imbalance_ratio=2.020111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=32429 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=64857 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[33097, 32765, 32105, 32768, 32762, 32740, 33096, 32811] avg=32768.0000 max=33097 max_rank=0 min=32105 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55085 max_load_reduction_pct=0.624674 moved_to_replicas=55086 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 19, 65525, 0, 32808, 65524, 32768] avg=32768.0000 max=65525 max_rank=3 min=0 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=67925 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] avg=32768.0000 max=33096 max_rank=3 min=32124 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33099 max_load_reduction_pct=0.500023 moved_to_replicas=65560 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] avg=32768.0000 max=33096 max_rank=3 min=32124 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33099 max_load_reduction_pct=0.500023 moved_to_replicas=65560 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 65557, 4, 66195, 32766, 2] avg=32768.0000 max=66195 max_rank=5 min=2 min_rank=7 imbalance_ratio=2.020111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[32550, 0, 65497, 36177, 29536, 65539, 32736, 109] avg=32768.0000 max=65539 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=32429 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=64857 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 65557, 4, 66195, 32766, 2] avg=32768.0000 max=66195 max_rank=5 min=2 min_rank=7 imbalance_ratio=2.020111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32802, 135387, 32738, 6, 3129, 47089, 10897, 96] avg=32768.0000 max=135387 max_rank=1 min=6 min_rank=3 imbalance_ratio=4.131683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[2670, 30914, 32020, 10, 32768, 32766, 32771, 98225] avg=32768.0000 max=98225 max_rank=7 min=10 min_rank=3 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] avg=32768.0000 max=33096 max_rank=3 min=32124 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33099 max_load_reduction_pct=0.500023 moved_to_replicas=65560 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 19, 65525, 0, 32808, 65524, 32768] avg=32768.0000 max=65525 max_rank=3 min=0 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[2670, 30914, 32020, 10, 32768, 32766, 32771, 98225] avg=32768.0000 max=98225 max_rank=7 min=10 min_rank=3 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=67925 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] avg=32768.0000 max=33096 max_rank=3 min=32124 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33099 max_load_reduction_pct=0.500023 moved_to_replicas=65560 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32802, 33096, 32738, 32771, 33096, 33096, 31684, 32861] avg=32768.0000 max=33096 max_rank=1 min=31684 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102291 max_load_reduction_pct=0.755545 moved_to_replicas=116284 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 65557, 4, 66195, 32766, 2] avg=32768.0000 max=66195 max_rank=5 min=2 min_rank=7 imbalance_ratio=2.020111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32020 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=65129 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=32429 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=64857 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32020 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=65129 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[2670, 30914, 32020, 10, 32768, 32766, 32771, 98225] avg=32768.0000 max=98225 max_rank=7 min=10 min_rank=3 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] avg=32768.0000 max=33096 max_rank=3 min=32124 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33099 max_load_reduction_pct=0.500023 moved_to_replicas=65560 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[2670, 30914, 32020, 10, 32768, 32766, 32771, 98225] avg=32768.0000 max=98225 max_rank=7 min=10 min_rank=3 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 19, 65525, 0, 32808, 65524, 32768] avg=32768.0000 max=65525 max_rank=3 min=0 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[32550, 0, 65497, 36177, 29536, 65539, 32736, 109] avg=32768.0000 max=65539 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[4, 32861, 312, 33224, 32678, 0, 32768, 130297] avg=32768.0000 max=130297 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.976349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32020 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=65129 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 65557, 4, 66195, 32766, 2] avg=32768.0000 max=66195 max_rank=5 min=2 min_rank=7 imbalance_ratio=2.020111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[4, 32861, 312, 33224, 32678, 0, 32768, 130297] avg=32768.0000 max=130297 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.976349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32020 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=65129 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=32429 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=64857 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[32550, 32443, 33096, 33096, 32617, 33096, 32736, 32510] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=67925 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[2670, 30914, 32020, 10, 32768, 32766, 32771, 98225] avg=32768.0000 max=98225 max_rank=7 min=10 min_rank=3 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] avg=32768.0000 max=33096 max_rank=3 min=32113 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.745996 moved_to_replicas=97329 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] avg=32768.0000 max=33096 max_rank=3 min=32124 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33099 max_load_reduction_pct=0.500023 moved_to_replicas=65560 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] avg=32768.0000 max=33096 max_rank=3 min=32113 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.745996 moved_to_replicas=97329 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[4, 32861, 312, 33224, 32678, 0, 32768, 130297] avg=32768.0000 max=130297 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.976349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32020 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=65129 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[4, 32861, 312, 33224, 32678, 0, 32768, 130297] avg=32768.0000 max=130297 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.976349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 65557, 4, 66195, 32766, 2] avg=32768.0000 max=66195 max_rank=5 min=2 min_rank=7 imbalance_ratio=2.020111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 19, 65525, 0, 32808, 65524, 32768] avg=32768.0000 max=65525 max_rank=3 min=0 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[1995, 97640, 55, 45357, 65513, 0, 51479, 105] avg=32768.0000 max=97640 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] avg=32768.0000 max=33096 max_rank=3 min=32113 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.745996 moved_to_replicas=97329 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[2670, 30914, 32020, 10, 32768, 32766, 32771, 98225] avg=32768.0000 max=98225 max_rank=7 min=10 min_rank=3 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[1995, 97640, 55, 45357, 65513, 0, 51479, 105] avg=32768.0000 max=97640 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] avg=32768.0000 max=33096 max_rank=3 min=32113 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.745996 moved_to_replicas=97329 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] avg=32768.0000 max=33096 max_rank=3 min=32124 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33099 max_load_reduction_pct=0.500023 moved_to_replicas=65560 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[32772, 32728, 32447, 33096, 32429, 32808, 33096, 32768] avg=32768.0000 max=33096 max_rank=3 min=32429 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=64857 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[4, 32861, 312, 33224, 32678, 0, 32768, 130297] avg=32768.0000 max=130297 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.976349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] avg=32768.0000 max=33097 max_rank=6 min=31834 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64543 max_load_reduction_pct=0.661030 moved_to_replicas=127605 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32020 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=65129 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] avg=32768.0000 max=33097 max_rank=6 min=31834 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64543 max_load_reduction_pct=0.661030 moved_to_replicas=127605 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[1995, 97640, 55, 45357, 65513, 0, 51479, 105] avg=32768.0000 max=97640 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] avg=32768.0000 max=33096 max_rank=3 min=32113 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.745996 moved_to_replicas=97329 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[1995, 97640, 55, 45357, 65513, 0, 51479, 105] avg=32768.0000 max=97640 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 65557, 4, 66195, 32766, 2] avg=32768.0000 max=66195 max_rank=5 min=2 min_rank=7 imbalance_ratio=2.020111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[2670, 30914, 32020, 10, 32768, 32766, 32771, 98225] avg=32768.0000 max=98225 max_rank=7 min=10 min_rank=3 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[138, 2, 32766, 65448, 32824, 53225, 65522, 12219] avg=32768.0000 max=65522 max_rank=6 min=2 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] avg=32768.0000 max=33097 max_rank=6 min=31834 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64543 max_load_reduction_pct=0.661030 moved_to_replicas=127605 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[138, 2, 32766, 65448, 32824, 53225, 65522, 12219] avg=32768.0000 max=65522 max_rank=6 min=2 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[4, 32861, 312, 33224, 32678, 0, 32768, 130297] avg=32768.0000 max=130297 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.976349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] avg=32768.0000 max=33097 max_rank=6 min=31834 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64543 max_load_reduction_pct=0.661030 moved_to_replicas=127605 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[1995, 97640, 55, 45357, 65513, 0, 51479, 105] avg=32768.0000 max=97640 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[32744, 32124, 32752, 33096, 32799, 33096, 32766, 32767] avg=32768.0000 max=33096 max_rank=3 min=32124 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33099 max_load_reduction_pct=0.500023 moved_to_replicas=65560 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32020 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=65129 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] avg=32768.0000 max=33096 max_rank=3 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=84907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] avg=32768.0000 max=33096 max_rank=3 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=84907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[138, 2, 32766, 65448, 32824, 53225, 65522, 12219] avg=32768.0000 max=65522 max_rank=6 min=2 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] avg=32768.0000 max=33096 max_rank=3 min=32113 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.745996 moved_to_replicas=97329 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] avg=32768.0000 max=33097 max_rank=6 min=31834 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64543 max_load_reduction_pct=0.661030 moved_to_replicas=127605 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[138, 2, 32766, 65448, 32824, 53225, 65522, 12219] avg=32768.0000 max=65522 max_rank=6 min=2 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[2670, 30914, 32020, 10, 32768, 32766, 32771, 98225] avg=32768.0000 max=98225 max_rank=7 min=10 min_rank=3 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[32764, 70448, 32820, 887, 59679, 11, 32763, 32772] avg=32768.0000 max=70448 max_rank=1 min=11 min_rank=5 imbalance_ratio=2.149902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[4, 32861, 312, 33224, 32678, 0, 32768, 130297] avg=32768.0000 max=130297 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.976349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] avg=32768.0000 max=33096 max_rank=3 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=84907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[32764, 70448, 32820, 887, 59679, 11, 32763, 32772] avg=32768.0000 max=70448 max_rank=1 min=11 min_rank=5 imbalance_ratio=2.149902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] avg=32768.0000 max=33096 max_rank=3 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=84907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[138, 2, 32766, 65448, 32824, 53225, 65522, 12219] avg=32768.0000 max=65522 max_rank=6 min=2 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[1995, 97640, 55, 45357, 65513, 0, 51479, 105] avg=32768.0000 max=97640 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] avg=32768.0000 max=33097 max_rank=4 min=32058 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37351 max_load_reduction_pct=0.530192 moved_to_replicas=63935 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[33096, 32852, 32020, 32775, 32768, 32766, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32020 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=65129 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] avg=32768.0000 max=33096 max_rank=3 min=32113 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.745996 moved_to_replicas=97329 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] avg=32768.0000 max=33097 max_rank=4 min=32058 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37351 max_load_reduction_pct=0.530192 moved_to_replicas=63935 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[32764, 70448, 32820, 887, 59679, 11, 32763, 32772] avg=32768.0000 max=70448 max_rank=1 min=11 min_rank=5 imbalance_ratio=2.149902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] avg=32768.0000 max=33096 max_rank=3 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=84907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] avg=32768.0000 max=33097 max_rank=6 min=31834 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64543 max_load_reduction_pct=0.661030 moved_to_replicas=127605 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[32764, 70448, 32820, 887, 59679, 11, 32763, 32772] avg=32768.0000 max=70448 max_rank=1 min=11 min_rank=5 imbalance_ratio=2.149902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[2, 1, 32768, 32954, 20, 96191, 34675, 65533] avg=32768.0000 max=96191 max_rank=5 min=1 min_rank=1 imbalance_ratio=2.935516 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] avg=32768.0000 max=33097 max_rank=4 min=32058 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37351 max_load_reduction_pct=0.530192 moved_to_replicas=63935 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[4, 32861, 312, 33224, 32678, 0, 32768, 130297] avg=32768.0000 max=130297 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.976349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[1995, 97640, 55, 45357, 65513, 0, 51479, 105] avg=32768.0000 max=97640 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[2, 1, 32768, 32954, 20, 96191, 34675, 65533] avg=32768.0000 max=96191 max_rank=5 min=1 min_rank=1 imbalance_ratio=2.935516 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] avg=32768.0000 max=33097 max_rank=4 min=32058 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37351 max_load_reduction_pct=0.530192 moved_to_replicas=63935 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[32764, 70448, 32820, 887, 59679, 11, 32763, 32772] avg=32768.0000 max=70448 max_rank=1 min=11 min_rank=5 imbalance_ratio=2.149902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[138, 2, 32766, 65448, 32824, 53225, 65522, 12219] avg=32768.0000 max=65522 max_rank=6 min=2 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31911 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63095 max_load_reduction_pct=0.655935 moved_to_replicas=97111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[32767, 32861, 32113, 33096, 32678, 32765, 32768, 33096] avg=32768.0000 max=33096 max_rank=3 min=32113 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.745996 moved_to_replicas=97329 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] avg=32768.0000 max=33097 max_rank=6 min=31834 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64543 max_load_reduction_pct=0.661030 moved_to_replicas=127605 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31911 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63095 max_load_reduction_pct=0.655935 moved_to_replicas=97111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[2, 1, 32768, 32954, 20, 96191, 34675, 65533] avg=32768.0000 max=96191 max_rank=5 min=1 min_rank=1 imbalance_ratio=2.935516 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] avg=32768.0000 max=33097 max_rank=4 min=32058 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37351 max_load_reduction_pct=0.530192 moved_to_replicas=63935 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] avg=32768.0000 max=33096 max_rank=3 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=84907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[2, 1, 32768, 32954, 20, 96191, 34675, 65533] avg=32768.0000 max=96191 max_rank=5 min=1 min_rank=1 imbalance_ratio=2.935516 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31911 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63095 max_load_reduction_pct=0.655935 moved_to_replicas=97111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=25 tokens_per_ep_rank=[2, 32599, 57173, 65469, 98261, 2, 8633, 5] avg=32768.0000 max=98261 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[138, 2, 32766, 65448, 32824, 53225, 65522, 12219] avg=32768.0000 max=65522 max_rank=6 min=2 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[1995, 97640, 55, 45357, 65513, 0, 51479, 105] avg=32768.0000 max=97640 max_rank=1 min=0 min_rank=5 imbalance_ratio=2.979736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=25 tokens_per_ep_rank=[2, 32599, 57173, 65469, 98261, 2, 8633, 5] avg=32768.0000 max=98261 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31911 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63095 max_load_reduction_pct=0.655935 moved_to_replicas=97111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[2, 1, 32768, 32954, 20, 96191, 34675, 65533] avg=32768.0000 max=96191 max_rank=5 min=1 min_rank=1 imbalance_ratio=2.935516 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[32764, 70448, 32820, 887, 59679, 11, 32763, 32772] avg=32768.0000 max=70448 max_rank=1 min=11 min_rank=5 imbalance_ratio=2.149902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=25 tokens_per_ep_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] avg=32768.0000 max=33099 max_rank=2 min=32378 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=65162 max_load_reduction_pct=0.663152 moved_to_replicas=121615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] avg=32768.0000 max=33096 max_rank=3 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=84907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[32638, 33096, 31834, 33096, 33096, 32765, 33097, 32522] avg=32768.0000 max=33097 max_rank=6 min=31834 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=64543 max_load_reduction_pct=0.661030 moved_to_replicas=127605 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=25 tokens_per_ep_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] avg=32768.0000 max=33099 max_rank=2 min=32378 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=65162 max_load_reduction_pct=0.663152 moved_to_replicas=121615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=25 tokens_per_ep_rank=[2, 32599, 57173, 65469, 98261, 2, 8633, 5] avg=32768.0000 max=98261 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31911 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63095 max_load_reduction_pct=0.655935 moved_to_replicas=97111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] avg=32768.0000 max=33097 max_rank=4 min=32058 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37351 max_load_reduction_pct=0.530192 moved_to_replicas=63935 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=25 tokens_per_ep_rank=[2, 32599, 57173, 65469, 98261, 2, 8633, 5] avg=32768.0000 max=98261 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=25 tokens_per_ep_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] avg=32768.0000 max=33099 max_rank=2 min=32378 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=65162 max_load_reduction_pct=0.663152 moved_to_replicas=121615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 1, 65980, 65499, 2] avg=32768.0000 max=65980 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.013550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[32764, 70448, 32820, 887, 59679, 11, 32763, 32772] avg=32768.0000 max=70448 max_rank=1 min=11 min_rank=5 imbalance_ratio=2.149902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[138, 2, 32766, 65448, 32824, 53225, 65522, 12219] avg=32768.0000 max=65522 max_rank=6 min=2 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 1, 65980, 65499, 2] avg=32768.0000 max=65980 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.013550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=25 tokens_per_ep_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] avg=32768.0000 max=33099 max_rank=2 min=32378 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=65162 max_load_reduction_pct=0.663152 moved_to_replicas=121615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=25 tokens_per_ep_rank=[2, 32599, 57173, 65469, 98261, 2, 8633, 5] avg=32768.0000 max=98261 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] avg=32768.0000 max=33096 max_rank=5 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32884 max_load_reduction_pct=0.498393 moved_to_replicas=65287 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[2, 1, 32768, 32954, 20, 96191, 34675, 65533] avg=32768.0000 max=96191 max_rank=5 min=1 min_rank=1 imbalance_ratio=2.935516 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] avg=32768.0000 max=33097 max_rank=4 min=32058 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37351 max_load_reduction_pct=0.530192 moved_to_replicas=63935 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[32490, 32428, 32766, 33096, 32824, 33096, 33096, 32348] avg=32768.0000 max=33096 max_rank=3 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=84907 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] avg=32768.0000 max=33096 max_rank=5 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32884 max_load_reduction_pct=0.498393 moved_to_replicas=65287 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 1, 65980, 65499, 2] avg=32768.0000 max=65980 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.013550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=25 tokens_per_ep_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] avg=32768.0000 max=33099 max_rank=2 min=32378 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=65162 max_load_reduction_pct=0.663152 moved_to_replicas=121615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31911 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63095 max_load_reduction_pct=0.655935 moved_to_replicas=97111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 1, 65980, 65499, 2] avg=32768.0000 max=65980 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.013550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=27 tokens_per_ep_rank=[4, 33092, 727, 18774, 65140, 32754, 77047, 34606] avg=32768.0000 max=77047 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.351288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] avg=32768.0000 max=33096 max_rank=5 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32884 max_load_reduction_pct=0.498393 moved_to_replicas=65287 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[2, 1, 32768, 32954, 20, 96191, 34675, 65533] avg=32768.0000 max=96191 max_rank=5 min=1 min_rank=1 imbalance_ratio=2.935516 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[32764, 70448, 32820, 887, 59679, 11, 32763, 32772] avg=32768.0000 max=70448 max_rank=1 min=11 min_rank=5 imbalance_ratio=2.149902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=27 tokens_per_ep_rank=[4, 33092, 727, 18774, 65140, 32754, 77047, 34606] avg=32768.0000 max=77047 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.351288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] avg=32768.0000 max=33096 max_rank=5 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32884 max_load_reduction_pct=0.498393 moved_to_replicas=65287 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 1, 65980, 65499, 2] avg=32768.0000 max=65980 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.013550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=27 tokens_per_ep_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=43951 max_load_reduction_pct=0.570444 moved_to_replicas=77505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=25 tokens_per_ep_rank=[2, 32599, 57173, 65469, 98261, 2, 8633, 5] avg=32768.0000 max=98261 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31911 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63095 max_load_reduction_pct=0.655935 moved_to_replicas=97111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[32764, 33096, 32820, 32058, 33097, 32774, 32763, 32772] avg=32768.0000 max=33097 max_rank=4 min=32058 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37351 max_load_reduction_pct=0.530192 moved_to_replicas=63935 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=27 tokens_per_ep_rank=[4, 33092, 727, 18774, 65140, 32754, 77047, 34606] avg=32768.0000 max=77047 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.351288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=27 tokens_per_ep_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=43951 max_load_reduction_pct=0.570444 moved_to_replicas=77505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] avg=32768.0000 max=33096 max_rank=5 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32884 max_load_reduction_pct=0.498393 moved_to_replicas=65287 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=25 tokens_per_ep_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] avg=32768.0000 max=33099 max_rank=2 min=32378 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=65162 max_load_reduction_pct=0.663152 moved_to_replicas=121615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=27 tokens_per_ep_rank=[4, 33092, 727, 18774, 65140, 32754, 77047, 34606] avg=32768.0000 max=77047 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.351288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=27 tokens_per_ep_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=43951 max_load_reduction_pct=0.570444 moved_to_replicas=77505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=28 tokens_per_ep_rank=[72324, 33304, 64048, 13596, 32766, 57, 1105, 44944] avg=32768.0000 max=72324 max_rank=0 min=57 min_rank=5 imbalance_ratio=2.207153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[2, 1, 32768, 32954, 20, 96191, 34675, 65533] avg=32768.0000 max=96191 max_rank=5 min=1 min_rank=1 imbalance_ratio=2.935516 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=25 tokens_per_ep_rank=[2, 32599, 57173, 65469, 98261, 2, 8633, 5] avg=32768.0000 max=98261 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=28 tokens_per_ep_rank=[72324, 33304, 64048, 13596, 32766, 57, 1105, 44944] avg=32768.0000 max=72324 max_rank=0 min=57 min_rank=5 imbalance_ratio=2.207153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=27 tokens_per_ep_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=43951 max_load_reduction_pct=0.570444 moved_to_replicas=77505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=27 tokens_per_ep_rank=[4, 33092, 727, 18774, 65140, 32754, 77047, 34606] avg=32768.0000 max=77047 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.351288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31075 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39227 max_load_reduction_pct=0.542379 moved_to_replicas=82236 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 1, 65980, 65499, 2] avg=32768.0000 max=65980 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.013550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[31911, 32766, 32768, 32954, 32457, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=31911 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63095 max_load_reduction_pct=0.655935 moved_to_replicas=97111 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=28 tokens_per_ep_rank=[72324, 33304, 64048, 13596, 32766, 57, 1105, 44944] avg=32768.0000 max=72324 max_rank=0 min=57 min_rank=5 imbalance_ratio=2.207153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=25 tokens_per_ep_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] avg=32768.0000 max=33099 max_rank=2 min=32378 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=65162 max_load_reduction_pct=0.663152 moved_to_replicas=121615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31075 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39227 max_load_reduction_pct=0.542379 moved_to_replicas=82236 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=27 tokens_per_ep_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=43951 max_load_reduction_pct=0.570444 moved_to_replicas=77505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] avg=32768.0000 max=33096 max_rank=5 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32884 max_load_reduction_pct=0.498393 moved_to_replicas=65287 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=28 tokens_per_ep_rank=[72324, 33304, 64048, 13596, 32766, 57, 1105, 44944] avg=32768.0000 max=72324 max_rank=0 min=57 min_rank=5 imbalance_ratio=2.207153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31075 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39227 max_load_reduction_pct=0.542379 moved_to_replicas=82236 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=29 tokens_per_ep_rank=[292, 98273, 5, 37, 35650, 1075, 32768, 94044] avg=32768.0000 max=98273 max_rank=1 min=5 min_rank=2 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=25 tokens_per_ep_rank=[2, 32599, 57173, 65469, 98261, 2, 8633, 5] avg=32768.0000 max=98261 max_rank=4 min=2 min_rank=0 imbalance_ratio=2.998688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 1, 65980, 65499, 2] avg=32768.0000 max=65980 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.013550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=29 tokens_per_ep_rank=[292, 98273, 5, 37, 35650, 1075, 32768, 94044] avg=32768.0000 max=98273 max_rank=1 min=5 min_rank=2 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31075 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39227 max_load_reduction_pct=0.542379 moved_to_replicas=82236 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=28 tokens_per_ep_rank=[72324, 33304, 64048, 13596, 32766, 57, 1105, 44944] avg=32768.0000 max=72324 max_rank=0 min=57 min_rank=5 imbalance_ratio=2.207153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=29 tokens_per_ep_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31814 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=128679 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=27 tokens_per_ep_rank=[4, 33092, 727, 18774, 65140, 32754, 77047, 34606] avg=32768.0000 max=77047 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.351288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=25 tokens_per_ep_rank=[32767, 32599, 33099, 33096, 33096, 32402, 32707, 32378] avg=32768.0000 max=33099 max_rank=2 min=32378 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=65162 max_load_reduction_pct=0.663152 moved_to_replicas=121615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=29 tokens_per_ep_rank=[292, 98273, 5, 37, 35650, 1075, 32768, 94044] avg=32768.0000 max=98273 max_rank=1 min=5 min_rank=2 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] avg=32768.0000 max=33096 max_rank=5 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32884 max_load_reduction_pct=0.498393 moved_to_replicas=65287 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=29 tokens_per_ep_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31814 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=128679 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31075 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39227 max_load_reduction_pct=0.542379 moved_to_replicas=82236 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=27 tokens_per_ep_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=43951 max_load_reduction_pct=0.570444 moved_to_replicas=77505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=29 tokens_per_ep_rank=[292, 98273, 5, 37, 35650, 1075, 32768, 94044] avg=32768.0000 max=98273 max_rank=1 min=5 min_rank=2 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=29 tokens_per_ep_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31814 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=128679 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 12337, 32978, 46303, 32747, 58087, 14382] avg=32768.0000 max=58087 max_rank=6 min=12337 min_rank=2 imbalance_ratio=1.772675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 1, 65980, 65499, 2] avg=32768.0000 max=65980 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.013550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=27 tokens_per_ep_rank=[4, 33092, 727, 18774, 65140, 32754, 77047, 34606] avg=32768.0000 max=77047 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.351288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 12337, 32978, 46303, 32747, 58087, 14382] avg=32768.0000 max=58087 max_rank=6 min=12337 min_rank=2 imbalance_ratio=1.772675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=29 tokens_per_ep_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31814 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=128679 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=29 tokens_per_ep_rank=[292, 98273, 5, 37, 35650, 1075, 32768, 94044] avg=32768.0000 max=98273 max_rank=1 min=5 min_rank=2 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] avg=32768.0000 max=33096 max_rank=6 min=31823 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24991 max_load_reduction_pct=0.430234 moved_to_replicas=38198 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=28 tokens_per_ep_rank=[72324, 33304, 64048, 13596, 32766, 57, 1105, 44944] avg=32768.0000 max=72324 max_rank=0 min=57 min_rank=5 imbalance_ratio=2.207153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=26 tokens_per_ep_rank=[32851, 32784, 32767, 32260, 32766, 33096, 33096, 32524] avg=32768.0000 max=33096 max_rank=5 min=32260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32884 max_load_reduction_pct=0.498393 moved_to_replicas=65287 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 12337, 32978, 46303, 32747, 58087, 14382] avg=32768.0000 max=58087 max_rank=6 min=12337 min_rank=2 imbalance_ratio=1.772675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=27 tokens_per_ep_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=43951 max_load_reduction_pct=0.570444 moved_to_replicas=77505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] avg=32768.0000 max=33096 max_rank=6 min=31823 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24991 max_load_reduction_pct=0.430234 moved_to_replicas=38198 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=29 tokens_per_ep_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31814 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=128679 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31075 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39227 max_load_reduction_pct=0.542379 moved_to_replicas=82236 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 12337, 32978, 46303, 32747, 58087, 14382] avg=32768.0000 max=58087 max_rank=6 min=12337 min_rank=2 imbalance_ratio=1.772675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] avg=32768.0000 max=33096 max_rank=6 min=31823 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24991 max_load_reduction_pct=0.430234 moved_to_replicas=38198 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=31 tokens_per_ep_rank=[6, 98300, 2, 2, 29628, 32766, 23304, 78136] avg=32768.0000 max=98300 max_rank=1 min=2 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=27 tokens_per_ep_rank=[4, 33092, 727, 18774, 65140, 32754, 77047, 34606] avg=32768.0000 max=77047 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.351288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=28 tokens_per_ep_rank=[72324, 33304, 64048, 13596, 32766, 57, 1105, 44944] avg=32768.0000 max=72324 max_rank=0 min=57 min_rank=5 imbalance_ratio=2.207153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=31 tokens_per_ep_rank=[6, 98300, 2, 2, 29628, 32766, 23304, 78136] avg=32768.0000 max=98300 max_rank=1 min=2 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] avg=32768.0000 max=33096 max_rank=6 min=31823 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24991 max_load_reduction_pct=0.430234 moved_to_replicas=38198 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 12337, 32978, 46303, 32747, 58087, 14382] avg=32768.0000 max=58087 max_rank=6 min=12337 min_rank=2 imbalance_ratio=1.772675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=31 tokens_per_ep_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=32111 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65202 max_load_reduction_pct=0.663296 moved_to_replicas=110244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=29 tokens_per_ep_rank=[292, 98273, 5, 37, 35650, 1075, 32768, 94044] avg=32768.0000 max=98273 max_rank=1 min=5 min_rank=2 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=31 tokens_per_ep_rank=[6, 98300, 2, 2, 29628, 32766, 23304, 78136] avg=32768.0000 max=98300 max_rank=1 min=2 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=27 tokens_per_ep_rank=[32767, 33092, 33096, 31147, 33096, 32754, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=43951 max_load_reduction_pct=0.570444 moved_to_replicas=77505 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31075 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39227 max_load_reduction_pct=0.542379 moved_to_replicas=82236 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=31 tokens_per_ep_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=32111 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65202 max_load_reduction_pct=0.663296 moved_to_replicas=110244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] avg=32768.0000 max=33096 max_rank=6 min=31823 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24991 max_load_reduction_pct=0.430234 moved_to_replicas=38198 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=29 tokens_per_ep_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31814 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=128679 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=31 tokens_per_ep_rank=[6, 98300, 2, 2, 29628, 32766, 23304, 78136] avg=32768.0000 max=98300 max_rank=1 min=2 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=31 tokens_per_ep_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=32111 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65202 max_load_reduction_pct=0.663296 moved_to_replicas=110244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=32 tokens_per_ep_rank=[74, 8, 65393, 32793, 32768, 126578, 1, 4529] avg=32768.0000 max=126578 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.862854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=28 tokens_per_ep_rank=[72324, 33304, 64048, 13596, 32766, 57, 1105, 44944] avg=32768.0000 max=72324 max_rank=0 min=57 min_rank=5 imbalance_ratio=2.207153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=31 tokens_per_ep_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=32111 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65202 max_load_reduction_pct=0.663296 moved_to_replicas=110244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=32 tokens_per_ep_rank=[74, 8, 65393, 32793, 32768, 126578, 1, 4529] avg=32768.0000 max=126578 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.862854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=29 tokens_per_ep_rank=[292, 98273, 5, 37, 35650, 1075, 32768, 94044] avg=32768.0000 max=98273 max_rank=1 min=5 min_rank=2 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=31 tokens_per_ep_rank=[6, 98300, 2, 2, 29628, 32766, 23304, 78136] avg=32768.0000 max=98300 max_rank=1 min=2 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=32 tokens_per_ep_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31756 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93482 max_load_reduction_pct=0.738533 moved_to_replicas=125779 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 12337, 32978, 46303, 32747, 58087, 14382] avg=32768.0000 max=58087 max_rank=6 min=12337 min_rank=2 imbalance_ratio=1.772675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=32 tokens_per_ep_rank=[74, 8, 65393, 32793, 32768, 126578, 1, 4529] avg=32768.0000 max=126578 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.862854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=28 tokens_per_ep_rank=[33095, 33096, 33096, 31075, 32766, 32822, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31075 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=39227 max_load_reduction_pct=0.542379 moved_to_replicas=82236 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=32 tokens_per_ep_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31756 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93482 max_load_reduction_pct=0.738533 moved_to_replicas=125779 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=29 tokens_per_ep_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31814 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=128679 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=31 tokens_per_ep_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=32111 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65202 max_load_reduction_pct=0.663296 moved_to_replicas=110244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] avg=32768.0000 max=33096 max_rank=6 min=31823 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24991 max_load_reduction_pct=0.430234 moved_to_replicas=38198 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=32 tokens_per_ep_rank=[74, 8, 65393, 32793, 32768, 126578, 1, 4529] avg=32768.0000 max=126578 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.862854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=32 tokens_per_ep_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31756 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93482 max_load_reduction_pct=0.738533 moved_to_replicas=125779 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=33 tokens_per_ep_rank=[32777, 60577, 65531, 4, 2192, 42, 68253, 32768] avg=32768.0000 max=68253 max_rank=6 min=4 min_rank=3 imbalance_ratio=2.082916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=29 tokens_per_ep_rank=[292, 98273, 5, 37, 35650, 1075, 32768, 94044] avg=32768.0000 max=98273 max_rank=1 min=5 min_rank=2 imbalance_ratio=2.999054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=32 tokens_per_ep_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31756 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93482 max_load_reduction_pct=0.738533 moved_to_replicas=125779 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=33 tokens_per_ep_rank=[32777, 60577, 65531, 4, 2192, 42, 68253, 32768] avg=32768.0000 max=68253 max_rank=6 min=4 min_rank=3 imbalance_ratio=2.082916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 12337, 32978, 46303, 32747, 58087, 14382] avg=32768.0000 max=58087 max_rank=6 min=12337 min_rank=2 imbalance_ratio=1.772675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=32 tokens_per_ep_rank=[74, 8, 65393, 32793, 32768, 126578, 1, 4529] avg=32768.0000 max=126578 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.862854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=33 tokens_per_ep_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31446 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35157 max_load_reduction_pct=0.515098 moved_to_replicas=95073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=31 tokens_per_ep_rank=[6, 98300, 2, 2, 29628, 32766, 23304, 78136] avg=32768.0000 max=98300 max_rank=1 min=2 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=33 tokens_per_ep_rank=[32777, 60577, 65531, 4, 2192, 42, 68253, 32768] avg=32768.0000 max=68253 max_rank=6 min=4 min_rank=3 imbalance_ratio=2.082916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=29 tokens_per_ep_rank=[33055, 33096, 32770, 32449, 33096, 31814, 32768, 33096] avg=32768.0000 max=33096 max_rank=1 min=31814 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=128679 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=33 tokens_per_ep_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31446 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35157 max_load_reduction_pct=0.515098 moved_to_replicas=95073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=32 tokens_per_ep_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31756 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93482 max_load_reduction_pct=0.738533 moved_to_replicas=125779 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] avg=32768.0000 max=33096 max_rank=6 min=31823 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24991 max_load_reduction_pct=0.430234 moved_to_replicas=38198 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=31 tokens_per_ep_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=32111 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65202 max_load_reduction_pct=0.663296 moved_to_replicas=110244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=33 tokens_per_ep_rank=[32777, 60577, 65531, 4, 2192, 42, 68253, 32768] avg=32768.0000 max=68253 max_rank=6 min=4 min_rank=3 imbalance_ratio=2.082916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=33 tokens_per_ep_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31446 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35157 max_load_reduction_pct=0.515098 moved_to_replicas=95073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=34 tokens_per_ep_rank=[65310, 65533, 884, 2, 32116, 2, 0, 98297] avg=32768.0000 max=98297 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 12337, 32978, 46303, 32747, 58087, 14382] avg=32768.0000 max=58087 max_rank=6 min=12337 min_rank=2 imbalance_ratio=1.772675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=33 tokens_per_ep_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31446 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35157 max_load_reduction_pct=0.515098 moved_to_replicas=95073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=34 tokens_per_ep_rank=[65310, 65533, 884, 2, 32116, 2, 0, 98297] avg=32768.0000 max=98297 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=129852 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=33 tokens_per_ep_rank=[32777, 60577, 65531, 4, 2192, 42, 68253, 32768] avg=32768.0000 max=68253 max_rank=6 min=4 min_rank=3 imbalance_ratio=2.082916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=31 tokens_per_ep_rank=[6, 98300, 2, 2, 29628, 32766, 23304, 78136] avg=32768.0000 max=98300 max_rank=1 min=2 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=32 tokens_per_ep_rank=[74, 8, 65393, 32793, 32768, 126578, 1, 4529] avg=32768.0000 max=126578 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.862854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=34 tokens_per_ep_rank=[65310, 65533, 884, 2, 32116, 2, 0, 98297] avg=32768.0000 max=98297 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=30 tokens_per_ep_rank=[32768, 32542, 33095, 32978, 33095, 32747, 33096, 31823] avg=32768.0000 max=33096 max_rank=6 min=31823 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24991 max_load_reduction_pct=0.430234 moved_to_replicas=38198 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=129852 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=33 tokens_per_ep_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31446 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35157 max_load_reduction_pct=0.515098 moved_to_replicas=95073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=32 tokens_per_ep_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31756 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93482 max_load_reduction_pct=0.738533 moved_to_replicas=125779 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=31 tokens_per_ep_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=32111 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65202 max_load_reduction_pct=0.663296 moved_to_replicas=110244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=34 tokens_per_ep_rank=[65310, 65533, 884, 2, 32116, 2, 0, 98297] avg=32768.0000 max=98297 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=129852 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=35 tokens_per_ep_rank=[4, 56, 9, 131065, 32767, 496, 97747, 0] avg=32768.0000 max=131065 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=31 tokens_per_ep_rank=[6, 98300, 2, 2, 29628, 32766, 23304, 78136] avg=32768.0000 max=98300 max_rank=1 min=2 min_rank=2 imbalance_ratio=2.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=129852 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=35 tokens_per_ep_rank=[4, 56, 9, 131065, 32767, 496, 97747, 0] avg=32768.0000 max=131065 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=35 tokens_per_ep_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=32382 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97969 max_load_reduction_pct=0.747484 moved_to_replicas=162620 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=34 tokens_per_ep_rank=[65310, 65533, 884, 2, 32116, 2, 0, 98297] avg=32768.0000 max=98297 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=33 tokens_per_ep_rank=[32777, 60577, 65531, 4, 2192, 42, 68253, 32768] avg=32768.0000 max=68253 max_rank=6 min=4 min_rank=3 imbalance_ratio=2.082916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=32 tokens_per_ep_rank=[74, 8, 65393, 32793, 32768, 126578, 1, 4529] avg=32768.0000 max=126578 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.862854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=35 tokens_per_ep_rank=[4, 56, 9, 131065, 32767, 496, 97747, 0] avg=32768.0000 max=131065 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=31 tokens_per_ep_rank=[32771, 33096, 32767, 32441, 32111, 32766, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=32111 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65202 max_load_reduction_pct=0.663296 moved_to_replicas=110244 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=35 tokens_per_ep_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=32382 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97969 max_load_reduction_pct=0.747484 moved_to_replicas=162620 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=129852 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=33 tokens_per_ep_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31446 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35157 max_load_reduction_pct=0.515098 moved_to_replicas=95073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=32 tokens_per_ep_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31756 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93482 max_load_reduction_pct=0.738533 moved_to_replicas=125779 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=35 tokens_per_ep_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=32382 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97969 max_load_reduction_pct=0.747484 moved_to_replicas=162620 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=35 tokens_per_ep_rank=[4, 56, 9, 131065, 32767, 496, 97747, 0] avg=32768.0000 max=131065 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=36 tokens_per_ep_rank=[98289, 32799, 2, 67487, 30669, 32770, 2, 126] avg=32768.0000 max=98289 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=35 tokens_per_ep_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=32382 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97969 max_load_reduction_pct=0.747484 moved_to_replicas=162620 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=36 tokens_per_ep_rank=[98289, 32799, 2, 67487, 30669, 32770, 2, 126] avg=32768.0000 max=98289 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=32 tokens_per_ep_rank=[74, 8, 65393, 32793, 32768, 126578, 1, 4529] avg=32768.0000 max=126578 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.862854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=35 tokens_per_ep_rank=[4, 56, 9, 131065, 32767, 496, 97747, 0] avg=32768.0000 max=131065 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=34 tokens_per_ep_rank=[65310, 65533, 884, 2, 32116, 2, 0, 98297] avg=32768.0000 max=98297 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] avg=32768.0000 max=33096 max_rank=0 min=32295 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=99584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=36 tokens_per_ep_rank=[98289, 32799, 2, 67487, 30669, 32770, 2, 126] avg=32768.0000 max=98289 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=33 tokens_per_ep_rank=[32777, 60577, 65531, 4, 2192, 42, 68253, 32768] avg=32768.0000 max=68253 max_rank=6 min=4 min_rank=3 imbalance_ratio=2.082916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] avg=32768.0000 max=33096 max_rank=0 min=32295 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=99584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=35 tokens_per_ep_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=32382 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97969 max_load_reduction_pct=0.747484 moved_to_replicas=162620 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=32 tokens_per_ep_rank=[31756, 32773, 33096, 32793, 32768, 33096, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31756 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93482 max_load_reduction_pct=0.738533 moved_to_replicas=125779 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=129852 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] avg=32768.0000 max=33096 max_rank=0 min=32295 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=99584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=33 tokens_per_ep_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31446 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35157 max_load_reduction_pct=0.515098 moved_to_replicas=95073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=36 tokens_per_ep_rank=[98289, 32799, 2, 67487, 30669, 32770, 2, 126] avg=32768.0000 max=98289 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 6, 0, 32770, 0, 65530, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 6, 0, 32770, 0, 65530, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] avg=32768.0000 max=33096 max_rank=0 min=32295 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=99584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=36 tokens_per_ep_rank=[98289, 32799, 2, 67487, 30669, 32770, 2, 126] avg=32768.0000 max=98289 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32428 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=97627 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=33 tokens_per_ep_rank=[32777, 60577, 65531, 4, 2192, 42, 68253, 32768] avg=32768.0000 max=68253 max_rank=6 min=4 min_rank=3 imbalance_ratio=2.082916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=35 tokens_per_ep_rank=[4, 56, 9, 131065, 32767, 496, 97747, 0] avg=32768.0000 max=131065 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 6, 0, 32770, 0, 65530, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=34 tokens_per_ep_rank=[65310, 65533, 884, 2, 32116, 2, 0, 98297] avg=32768.0000 max=98297 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32428 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=97627 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] avg=32768.0000 max=33096 max_rank=0 min=32295 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=99584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=33 tokens_per_ep_rank=[32777, 33096, 33096, 32769, 33096, 31446, 33096, 32768] avg=32768.0000 max=33096 max_rank=1 min=31446 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35157 max_load_reduction_pct=0.515098 moved_to_replicas=95073 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32428 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=97627 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 6, 0, 32770, 0, 65530, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=35 tokens_per_ep_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=32382 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97969 max_load_reduction_pct=0.747484 moved_to_replicas=162620 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=129852 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=38 tokens_per_ep_rank=[272, 0, 65349, 32763, 32537, 33502, 32752, 64969] avg=32768.0000 max=65349 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.994293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=38 tokens_per_ep_rank=[272, 0, 65349, 32763, 32537, 33502, 32752, 64969] avg=32768.0000 max=65349 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.994293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 6, 0, 32770, 0, 65530, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32428 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=97627 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=38 tokens_per_ep_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] avg=32768.0000 max=33096 max_rank=2 min=32253 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32253 max_load_reduction_pct=0.493550 moved_to_replicas=64532 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=34 tokens_per_ep_rank=[65310, 65533, 884, 2, 32116, 2, 0, 98297] avg=32768.0000 max=98297 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=38 tokens_per_ep_rank=[272, 0, 65349, 32763, 32537, 33502, 32752, 64969] avg=32768.0000 max=65349 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.994293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=36 tokens_per_ep_rank=[98289, 32799, 2, 67487, 30669, 32770, 2, 126] avg=32768.0000 max=98289 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=35 tokens_per_ep_rank=[4, 56, 9, 131065, 32767, 496, 97747, 0] avg=32768.0000 max=131065 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=38 tokens_per_ep_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] avg=32768.0000 max=33096 max_rank=2 min=32253 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32253 max_load_reduction_pct=0.493550 moved_to_replicas=64532 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32428 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=97627 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32438, 32118, 32439, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32118 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65201 max_load_reduction_pct=0.663306 moved_to_replicas=129852 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=38 tokens_per_ep_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] avg=32768.0000 max=33096 max_rank=2 min=32253 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32253 max_load_reduction_pct=0.493550 moved_to_replicas=64532 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=38 tokens_per_ep_rank=[272, 0, 65349, 32763, 32537, 33502, 32752, 64969] avg=32768.0000 max=65349 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.994293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] avg=32768.0000 max=33096 max_rank=0 min=32295 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=99584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=39 tokens_per_ep_rank=[47244, 0, 32767, 32772, 4, 98255, 32828, 18274] avg=32768.0000 max=98255 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=35 tokens_per_ep_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=32382 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97969 max_load_reduction_pct=0.747484 moved_to_replicas=162620 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=39 tokens_per_ep_rank=[47244, 0, 32767, 32772, 4, 98255, 32828, 18274] avg=32768.0000 max=98255 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=38 tokens_per_ep_rank=[272, 0, 65349, 32763, 32537, 33502, 32752, 64969] avg=32768.0000 max=65349 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.994293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=38 tokens_per_ep_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] avg=32768.0000 max=33096 max_rank=2 min=32253 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32253 max_load_reduction_pct=0.493550 moved_to_replicas=64532 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=39 tokens_per_ep_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] avg=32768.0000 max=33098 max_rank=0 min=32398 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65157 max_load_reduction_pct=0.663142 moved_to_replicas=79307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=39 tokens_per_ep_rank=[47244, 0, 32767, 32772, 4, 98255, 32828, 18274] avg=32768.0000 max=98255 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=35 tokens_per_ep_rank=[4, 56, 9, 131065, 32767, 496, 97747, 0] avg=32768.0000 max=131065 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 6, 0, 32770, 0, 65530, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=39 tokens_per_ep_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] avg=32768.0000 max=33098 max_rank=0 min=32398 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65157 max_load_reduction_pct=0.663142 moved_to_replicas=79307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=38 tokens_per_ep_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] avg=32768.0000 max=33096 max_rank=2 min=32253 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32253 max_load_reduction_pct=0.493550 moved_to_replicas=64532 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=36 tokens_per_ep_rank=[98289, 32799, 2, 67487, 30669, 32770, 2, 126] avg=32768.0000 max=98289 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=39 tokens_per_ep_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] avg=32768.0000 max=33098 max_rank=0 min=32398 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65157 max_load_reduction_pct=0.663142 moved_to_replicas=79307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=35 tokens_per_ep_rank=[32769, 32821, 32448, 33096, 32767, 32382, 33096, 32765] avg=32768.0000 max=33096 max_rank=3 min=32382 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97969 max_load_reduction_pct=0.747484 moved_to_replicas=162620 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=39 tokens_per_ep_rank=[47244, 0, 32767, 32772, 4, 98255, 32828, 18274] avg=32768.0000 max=98255 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=40 tokens_per_ep_rank=[96483, 65576, 16, 32769, 4, 32765, 34529, 2] avg=32768.0000 max=96483 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32428 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=97627 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] avg=32768.0000 max=33096 max_rank=0 min=32295 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=99584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=40 tokens_per_ep_rank=[96483, 65576, 16, 32769, 4, 32765, 34529, 2] avg=32768.0000 max=96483 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=39 tokens_per_ep_rank=[47244, 0, 32767, 32772, 4, 98255, 32828, 18274] avg=32768.0000 max=98255 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=39 tokens_per_ep_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] avg=32768.0000 max=33098 max_rank=0 min=32398 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65157 max_load_reduction_pct=0.663142 moved_to_replicas=79307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=32059 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63387 max_load_reduction_pct=0.656976 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=40 tokens_per_ep_rank=[96483, 65576, 16, 32769, 4, 32765, 34529, 2] avg=32768.0000 max=96483 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=36 tokens_per_ep_rank=[98289, 32799, 2, 67487, 30669, 32770, 2, 126] avg=32768.0000 max=98289 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=38 tokens_per_ep_rank=[272, 0, 65349, 32763, 32537, 33502, 32752, 64969] avg=32768.0000 max=65349 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.994293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=32059 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63387 max_load_reduction_pct=0.656976 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=39 tokens_per_ep_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] avg=32768.0000 max=33098 max_rank=0 min=32398 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65157 max_load_reduction_pct=0.663142 moved_to_replicas=79307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 6, 0, 32770, 0, 65530, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=32059 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63387 max_load_reduction_pct=0.656976 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=36 tokens_per_ep_rank=[33096, 32799, 32767, 33096, 32295, 32770, 32430, 32891] avg=32768.0000 max=33096 max_rank=0 min=32295 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=99584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=40 tokens_per_ep_rank=[96483, 65576, 16, 32769, 4, 32765, 34529, 2] avg=32768.0000 max=96483 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=38 tokens_per_ep_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] avg=32768.0000 max=33096 max_rank=2 min=32253 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32253 max_load_reduction_pct=0.493550 moved_to_replicas=64532 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=41 tokens_per_ep_rank=[4, 2, 31352, 35330, 4, 72503, 32569, 90380] avg=32768.0000 max=90380 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.758179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32428 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=97627 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=41 tokens_per_ep_rank=[4, 2, 31352, 35330, 4, 72503, 32569, 90380] avg=32768.0000 max=90380 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.758179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=40 tokens_per_ep_rank=[96483, 65576, 16, 32769, 4, 32765, 34529, 2] avg=32768.0000 max=96483 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=32059 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63387 max_load_reduction_pct=0.656976 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=41 tokens_per_ep_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] avg=32768.0000 max=33098 max_rank=5 min=31974 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=57282 max_load_reduction_pct=0.633791 moved_to_replicas=98925 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=41 tokens_per_ep_rank=[4, 2, 31352, 35330, 4, 72503, 32569, 90380] avg=32768.0000 max=90380 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.758179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 6, 0, 32770, 0, 65530, 98289] avg=32768.0000 max=98289 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=39 tokens_per_ep_rank=[47244, 0, 32767, 32772, 4, 98255, 32828, 18274] avg=32768.0000 max=98255 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=41 tokens_per_ep_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] avg=32768.0000 max=33098 max_rank=5 min=31974 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=57282 max_load_reduction_pct=0.633791 moved_to_replicas=98925 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=32059 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63387 max_load_reduction_pct=0.656976 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=38 tokens_per_ep_rank=[272, 0, 65349, 32763, 32537, 33502, 32752, 64969] avg=32768.0000 max=65349 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.994293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=41 tokens_per_ep_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] avg=32768.0000 max=33098 max_rank=5 min=31974 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=57282 max_load_reduction_pct=0.633791 moved_to_replicas=98925 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=37 tokens_per_ep_rank=[32766, 32783, 32440, 32765, 32770, 32428, 33096, 33096] avg=32768.0000 max=33096 max_rank=6 min=32428 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65193 max_load_reduction_pct=0.663279 moved_to_replicas=97627 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=41 tokens_per_ep_rank=[4, 2, 31352, 35330, 4, 72503, 32569, 90380] avg=32768.0000 max=90380 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.758179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=42 tokens_per_ep_rank=[65496, 16, 4, 32834, 65533, 65532, 32728, 1] avg=32768.0000 max=65533 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=39 tokens_per_ep_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] avg=32768.0000 max=33098 max_rank=0 min=32398 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65157 max_load_reduction_pct=0.663142 moved_to_replicas=79307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=38 tokens_per_ep_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] avg=32768.0000 max=33096 max_rank=2 min=32253 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32253 max_load_reduction_pct=0.493550 moved_to_replicas=64532 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=42 tokens_per_ep_rank=[65496, 16, 4, 32834, 65533, 65532, 32728, 1] avg=32768.0000 max=65533 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=41 tokens_per_ep_rank=[4, 2, 31352, 35330, 4, 72503, 32569, 90380] avg=32768.0000 max=90380 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.758179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=41 tokens_per_ep_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] avg=32768.0000 max=33098 max_rank=5 min=31974 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=57282 max_load_reduction_pct=0.633791 moved_to_replicas=98925 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=42 tokens_per_ep_rank=[65496, 16, 4, 32834, 65533, 65532, 32728, 1] avg=32768.0000 max=65533 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] avg=32768.0000 max=33096 max_rank=0 min=32416 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97273 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=38 tokens_per_ep_rank=[272, 0, 65349, 32763, 32537, 33502, 32752, 64969] avg=32768.0000 max=65349 max_rank=2 min=0 min_rank=1 imbalance_ratio=1.994293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] avg=32768.0000 max=33096 max_rank=0 min=32416 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97273 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=40 tokens_per_ep_rank=[96483, 65576, 16, 32769, 4, 32765, 34529, 2] avg=32768.0000 max=96483 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=41 tokens_per_ep_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] avg=32768.0000 max=33098 max_rank=5 min=31974 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=57282 max_load_reduction_pct=0.633791 moved_to_replicas=98925 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=39 tokens_per_ep_rank=[47244, 0, 32767, 32772, 4, 98255, 32828, 18274] avg=32768.0000 max=98255 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] avg=32768.0000 max=33096 max_rank=0 min=32416 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97273 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=38 tokens_per_ep_rank=[32551, 32253, 33096, 32763, 32537, 33096, 32752, 33096] avg=32768.0000 max=33096 max_rank=2 min=32253 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32253 max_load_reduction_pct=0.493550 moved_to_replicas=64532 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=42 tokens_per_ep_rank=[65496, 16, 4, 32834, 65533, 65532, 32728, 1] avg=32768.0000 max=65533 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=32059 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63387 max_load_reduction_pct=0.656976 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=43 tokens_per_ep_rank=[16, 33519, 32635, 140, 32818, 33045, 32746, 97225] avg=32768.0000 max=97225 max_rank=7 min=16 min_rank=0 imbalance_ratio=2.967072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=39 tokens_per_ep_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] avg=32768.0000 max=33098 max_rank=0 min=32398 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65157 max_load_reduction_pct=0.663142 moved_to_replicas=79307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=43 tokens_per_ep_rank=[16, 33519, 32635, 140, 32818, 33045, 32746, 97225] avg=32768.0000 max=97225 max_rank=7 min=16 min_rank=0 imbalance_ratio=2.967072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=42 tokens_per_ep_rank=[65496, 16, 4, 32834, 65533, 65532, 32728, 1] avg=32768.0000 max=65533 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=43 tokens_per_ep_rank=[16, 33519, 32635, 140, 32818, 33045, 32746, 97225] avg=32768.0000 max=97225 max_rank=7 min=16 min_rank=0 imbalance_ratio=2.967072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] avg=32768.0000 max=33096 max_rank=0 min=32416 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97273 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=43 tokens_per_ep_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] avg=32768.0000 max=33096 max_rank=1 min=31927 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64129 max_load_reduction_pct=0.659594 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=39 tokens_per_ep_rank=[47244, 0, 32767, 32772, 4, 98255, 32828, 18274] avg=32768.0000 max=98255 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=41 tokens_per_ep_rank=[4, 2, 31352, 35330, 4, 72503, 32569, 90380] avg=32768.0000 max=90380 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.758179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=43 tokens_per_ep_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] avg=32768.0000 max=33096 max_rank=1 min=31927 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64129 max_load_reduction_pct=0.659594 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] avg=32768.0000 max=33096 max_rank=0 min=32416 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97273 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=43 tokens_per_ep_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] avg=32768.0000 max=33096 max_rank=1 min=31927 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64129 max_load_reduction_pct=0.659594 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=40 tokens_per_ep_rank=[96483, 65576, 16, 32769, 4, 32765, 34529, 2] avg=32768.0000 max=96483 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=39 tokens_per_ep_rank=[33098, 32765, 32767, 32772, 32398, 33096, 32828, 32420] avg=32768.0000 max=33098 max_rank=0 min=32398 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=65157 max_load_reduction_pct=0.663142 moved_to_replicas=79307 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=43 tokens_per_ep_rank=[16, 33519, 32635, 140, 32818, 33045, 32746, 97225] avg=32768.0000 max=97225 max_rank=7 min=16 min_rank=0 imbalance_ratio=2.967072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=41 tokens_per_ep_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] avg=32768.0000 max=33098 max_rank=5 min=31974 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=57282 max_load_reduction_pct=0.633791 moved_to_replicas=98925 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=44 tokens_per_ep_rank=[29, 32727, 102904, 60174, 407, 32794, 21, 33088] avg=32768.0000 max=102904 max_rank=2 min=21 min_rank=6 imbalance_ratio=3.140381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=44 tokens_per_ep_rank=[29, 32727, 102904, 60174, 407, 32794, 21, 33088] avg=32768.0000 max=102904 max_rank=2 min=21 min_rank=6 imbalance_ratio=3.140381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=43 tokens_per_ep_rank=[16, 33519, 32635, 140, 32818, 33045, 32746, 97225] avg=32768.0000 max=97225 max_rank=7 min=16 min_rank=0 imbalance_ratio=2.967072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=32059 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63387 max_load_reduction_pct=0.656976 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=44 tokens_per_ep_rank=[29, 32727, 102904, 60174, 407, 32794, 21, 33088] avg=32768.0000 max=102904 max_rank=2 min=21 min_rank=6 imbalance_ratio=3.140381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=43 tokens_per_ep_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] avg=32768.0000 max=33096 max_rank=1 min=31927 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64129 max_load_reduction_pct=0.659594 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=44 tokens_per_ep_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] avg=32768.0000 max=33095 max_rank=2 min=31766 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=69809 max_load_reduction_pct=0.678390 moved_to_replicas=96886 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=40 tokens_per_ep_rank=[96483, 65576, 16, 32769, 4, 32765, 34529, 2] avg=32768.0000 max=96483 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=44 tokens_per_ep_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] avg=32768.0000 max=33095 max_rank=2 min=31766 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=69809 max_load_reduction_pct=0.678390 moved_to_replicas=96886 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=42 tokens_per_ep_rank=[65496, 16, 4, 32834, 65533, 65532, 32728, 1] avg=32768.0000 max=65533 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=43 tokens_per_ep_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] avg=32768.0000 max=33096 max_rank=1 min=31927 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64129 max_load_reduction_pct=0.659594 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=44 tokens_per_ep_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] avg=32768.0000 max=33095 max_rank=2 min=31766 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=69809 max_load_reduction_pct=0.678390 moved_to_replicas=96886 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=41 tokens_per_ep_rank=[4, 2, 31352, 35330, 4, 72503, 32569, 90380] avg=32768.0000 max=90380 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.758179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=40 tokens_per_ep_rank=[33096, 33096, 32496, 32769, 32059, 32765, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=32059 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63387 max_load_reduction_pct=0.656976 moved_to_replicas=97300 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=45 tokens_per_ep_rank=[5730, 65534, 98298, 0, 71, 32684, 27057, 32770] avg=32768.0000 max=98298 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=44 tokens_per_ep_rank=[29, 32727, 102904, 60174, 407, 32794, 21, 33088] avg=32768.0000 max=102904 max_rank=2 min=21 min_rank=6 imbalance_ratio=3.140381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] avg=32768.0000 max=33096 max_rank=0 min=32416 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97273 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=45 tokens_per_ep_rank=[5730, 65534, 98298, 0, 71, 32684, 27057, 32770] avg=32768.0000 max=98298 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=45 tokens_per_ep_rank=[5730, 65534, 98298, 0, 71, 32684, 27057, 32770] avg=32768.0000 max=98298 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=44 tokens_per_ep_rank=[29, 32727, 102904, 60174, 407, 32794, 21, 33088] avg=32768.0000 max=102904 max_rank=2 min=21 min_rank=6 imbalance_ratio=3.140381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=41 tokens_per_ep_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] avg=32768.0000 max=33098 max_rank=5 min=31974 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=57282 max_load_reduction_pct=0.633791 moved_to_replicas=98925 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=45 tokens_per_ep_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] avg=32768.0000 max=33097 max_rank=0 min=32128 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65201 max_load_reduction_pct=0.663299 moved_to_replicas=97640 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=44 tokens_per_ep_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] avg=32768.0000 max=33095 max_rank=2 min=31766 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=69809 max_load_reduction_pct=0.678390 moved_to_replicas=96886 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=41 tokens_per_ep_rank=[4, 2, 31352, 35330, 4, 72503, 32569, 90380] avg=32768.0000 max=90380 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.758179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=45 tokens_per_ep_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] avg=32768.0000 max=33097 max_rank=0 min=32128 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65201 max_load_reduction_pct=0.663299 moved_to_replicas=97640 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=43 tokens_per_ep_rank=[16, 33519, 32635, 140, 32818, 33045, 32746, 97225] avg=32768.0000 max=97225 max_rank=7 min=16 min_rank=0 imbalance_ratio=2.967072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=45 tokens_per_ep_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] avg=32768.0000 max=33097 max_rank=0 min=32128 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65201 max_load_reduction_pct=0.663299 moved_to_replicas=97640 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=44 tokens_per_ep_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] avg=32768.0000 max=33095 max_rank=2 min=31766 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=69809 max_load_reduction_pct=0.678390 moved_to_replicas=96886 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=41 tokens_per_ep_rank=[31974, 32446, 33096, 33096, 32769, 33098, 32569, 33096] avg=32768.0000 max=33098 max_rank=5 min=31974 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=57282 max_load_reduction_pct=0.633791 moved_to_replicas=98925 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=42 tokens_per_ep_rank=[65496, 16, 4, 32834, 65533, 65532, 32728, 1] avg=32768.0000 max=65533 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=46 tokens_per_ep_rank=[49559, 4, 48717, 14, 4, 98286, 32775, 32785] avg=32768.0000 max=98286 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=43 tokens_per_ep_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] avg=32768.0000 max=33096 max_rank=1 min=31927 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64129 max_load_reduction_pct=0.659594 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=45 tokens_per_ep_rank=[5730, 65534, 98298, 0, 71, 32684, 27057, 32770] avg=32768.0000 max=98298 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=46 tokens_per_ep_rank=[49559, 4, 48717, 14, 4, 98286, 32775, 32785] avg=32768.0000 max=98286 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=45 tokens_per_ep_rank=[5730, 65534, 98298, 0, 71, 32684, 27057, 32770] avg=32768.0000 max=98298 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=46 tokens_per_ep_rank=[49559, 4, 48717, 14, 4, 98286, 32775, 32785] avg=32768.0000 max=98286 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] avg=32768.0000 max=33096 max_rank=0 min=32416 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97273 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=46 tokens_per_ep_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] avg=32768.0000 max=33098 max_rank=0 min=32096 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65188 max_load_reduction_pct=0.663248 moved_to_replicas=97274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=45 tokens_per_ep_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] avg=32768.0000 max=33097 max_rank=0 min=32128 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65201 max_load_reduction_pct=0.663299 moved_to_replicas=97640 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=42 tokens_per_ep_rank=[65496, 16, 4, 32834, 65533, 65532, 32728, 1] avg=32768.0000 max=65533 max_rank=4 min=1 min_rank=7 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=46 tokens_per_ep_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] avg=32768.0000 max=33098 max_rank=0 min=32096 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65188 max_load_reduction_pct=0.663248 moved_to_replicas=97274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=44 tokens_per_ep_rank=[29, 32727, 102904, 60174, 407, 32794, 21, 33088] avg=32768.0000 max=102904 max_rank=2 min=21 min_rank=6 imbalance_ratio=3.140381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=45 tokens_per_ep_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] avg=32768.0000 max=33097 max_rank=0 min=32128 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65201 max_load_reduction_pct=0.663299 moved_to_replicas=97640 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=46 tokens_per_ep_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] avg=32768.0000 max=33098 max_rank=0 min=32096 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65188 max_load_reduction_pct=0.663248 moved_to_replicas=97274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=42 tokens_per_ep_rank=[33096, 32416, 32440, 32834, 33096, 33096, 32728, 32438] avg=32768.0000 max=33096 max_rank=0 min=32416 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=97273 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=47 tokens_per_ep_rank=[7643, 14643, 32768, 65534, 32767, 32763, 32764, 43262] avg=32768.0000 max=65534 max_rank=3 min=7643 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=43 tokens_per_ep_rank=[16, 33519, 32635, 140, 32818, 33045, 32746, 97225] avg=32768.0000 max=97225 max_rank=7 min=16 min_rank=0 imbalance_ratio=2.967072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=44 tokens_per_ep_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] avg=32768.0000 max=33095 max_rank=2 min=31766 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=69809 max_load_reduction_pct=0.678390 moved_to_replicas=96886 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=46 tokens_per_ep_rank=[49559, 4, 48717, 14, 4, 98286, 32775, 32785] avg=32768.0000 max=98286 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=47 tokens_per_ep_rank=[7643, 14643, 32768, 65534, 32767, 32763, 32764, 43262] avg=32768.0000 max=65534 max_rank=3 min=7643 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=46 tokens_per_ep_rank=[49559, 4, 48717, 14, 4, 98286, 32775, 32785] avg=32768.0000 max=98286 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=47 tokens_per_ep_rank=[7643, 14643, 32768, 65534, 32767, 32763, 32764, 43262] avg=32768.0000 max=65534 max_rank=3 min=7643 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=47 tokens_per_ep_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] avg=32768.0000 max=33096 max_rank=0 min=31795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=42604 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=43 tokens_per_ep_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] avg=32768.0000 max=33096 max_rank=1 min=31927 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64129 max_load_reduction_pct=0.659594 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=46 tokens_per_ep_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] avg=32768.0000 max=33098 max_rank=0 min=32096 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65188 max_load_reduction_pct=0.663248 moved_to_replicas=97274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=47 tokens_per_ep_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] avg=32768.0000 max=33096 max_rank=0 min=31795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=42604 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=43 tokens_per_ep_rank=[16, 33519, 32635, 140, 32818, 33045, 32746, 97225] avg=32768.0000 max=97225 max_rank=7 min=16 min_rank=0 imbalance_ratio=2.967072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=47 tokens_per_ep_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] avg=32768.0000 max=33096 max_rank=0 min=31795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=42604 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=46 tokens_per_ep_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] avg=32768.0000 max=33098 max_rank=0 min=32096 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65188 max_load_reduction_pct=0.663248 moved_to_replicas=97274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=45 tokens_per_ep_rank=[5730, 65534, 98298, 0, 71, 32684, 27057, 32770] avg=32768.0000 max=98298 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=48 tokens_per_ep_rank=[98086, 65515, 2, 4, 2, 68689, 29794, 52] avg=32768.0000 max=98086 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.993347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=43 tokens_per_ep_rank=[32781, 33096, 32635, 31927, 32818, 33045, 32746, 33096] avg=32768.0000 max=33096 max_rank=1 min=31927 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64129 max_load_reduction_pct=0.659594 moved_to_replicas=64552 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=44 tokens_per_ep_rank=[29, 32727, 102904, 60174, 407, 32794, 21, 33088] avg=32768.0000 max=102904 max_rank=2 min=21 min_rank=6 imbalance_ratio=3.140381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=47 tokens_per_ep_rank=[7643, 14643, 32768, 65534, 32767, 32763, 32764, 43262] avg=32768.0000 max=65534 max_rank=3 min=7643 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=48 tokens_per_ep_rank=[98086, 65515, 2, 4, 2, 68689, 29794, 52] avg=32768.0000 max=98086 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.993347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=45 tokens_per_ep_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] avg=32768.0000 max=33097 max_rank=0 min=32128 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65201 max_load_reduction_pct=0.663299 moved_to_replicas=97640 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=48 tokens_per_ep_rank=[98086, 65515, 2, 4, 2, 68689, 29794, 52] avg=32768.0000 max=98086 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.993347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=47 tokens_per_ep_rank=[7643, 14643, 32768, 65534, 32767, 32763, 32764, 43262] avg=32768.0000 max=65534 max_rank=3 min=7643 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] avg=32768.0000 max=33096 max_rank=0 min=32002 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64990 max_load_reduction_pct=0.662582 moved_to_replicas=133002 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=44 tokens_per_ep_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] avg=32768.0000 max=33095 max_rank=2 min=31766 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=69809 max_load_reduction_pct=0.678390 moved_to_replicas=96886 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] avg=32768.0000 max=33096 max_rank=0 min=32002 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64990 max_load_reduction_pct=0.662582 moved_to_replicas=133002 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=47 tokens_per_ep_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] avg=32768.0000 max=33096 max_rank=0 min=31795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=42604 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=44 tokens_per_ep_rank=[29, 32727, 102904, 60174, 407, 32794, 21, 33088] avg=32768.0000 max=102904 max_rank=2 min=21 min_rank=6 imbalance_ratio=3.140381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] avg=32768.0000 max=33096 max_rank=0 min=32002 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64990 max_load_reduction_pct=0.662582 moved_to_replicas=133002 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=47 tokens_per_ep_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] avg=32768.0000 max=33096 max_rank=0 min=31795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=42604 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=46 tokens_per_ep_rank=[49559, 4, 48717, 14, 4, 98286, 32775, 32785] avg=32768.0000 max=98286 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=45 tokens_per_ep_rank=[5730, 65534, 98298, 0, 71, 32684, 27057, 32770] avg=32768.0000 max=98298 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=44 tokens_per_ep_rank=[32794, 32727, 33095, 33094, 31766, 32794, 32786, 33088] avg=32768.0000 max=33095 max_rank=2 min=31766 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=69809 max_load_reduction_pct=0.678390 moved_to_replicas=96886 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=48 tokens_per_ep_rank=[98086, 65515, 2, 4, 2, 68689, 29794, 52] avg=32768.0000 max=98086 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.993347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=46 tokens_per_ep_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] avg=32768.0000 max=33098 max_rank=0 min=32096 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65188 max_load_reduction_pct=0.663248 moved_to_replicas=97274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=48 tokens_per_ep_rank=[98086, 65515, 2, 4, 2, 68689, 29794, 52] avg=32768.0000 max=98086 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.993347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=45 tokens_per_ep_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] avg=32768.0000 max=33097 max_rank=0 min=32128 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65201 max_load_reduction_pct=0.663299 moved_to_replicas=97640 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] avg=32768.0000 max=33096 max_rank=0 min=32002 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64990 max_load_reduction_pct=0.662582 moved_to_replicas=133002 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=45 tokens_per_ep_rank=[5730, 65534, 98298, 0, 71, 32684, 27057, 32770] avg=32768.0000 max=98298 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] avg=32768.0000 max=33096 max_rank=0 min=32002 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64990 max_load_reduction_pct=0.662582 moved_to_replicas=133002 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=47 tokens_per_ep_rank=[7643, 14643, 32768, 65534, 32767, 32763, 32764, 43262] avg=32768.0000 max=65534 max_rank=3 min=7643 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=46 tokens_per_ep_rank=[49559, 4, 48717, 14, 4, 98286, 32775, 32785] avg=32768.0000 max=98286 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=45 tokens_per_ep_rank=[33097, 33096, 33096, 32765, 32508, 32684, 32128, 32770] avg=32768.0000 max=33097 max_rank=0 min=32128 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=65201 max_load_reduction_pct=0.663299 moved_to_replicas=97640 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=47 tokens_per_ep_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] avg=32768.0000 max=33096 max_rank=0 min=31795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=42604 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=46 tokens_per_ep_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] avg=32768.0000 max=33098 max_rank=0 min=32096 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65188 max_load_reduction_pct=0.663248 moved_to_replicas=97274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=46 tokens_per_ep_rank=[49559, 4, 48717, 14, 4, 98286, 32775, 32785] avg=32768.0000 max=98286 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=48 tokens_per_ep_rank=[98086, 65515, 2, 4, 2, 68689, 29794, 52] avg=32768.0000 max=98086 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.993347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=47 tokens_per_ep_rank=[7643, 14643, 32768, 65534, 32767, 32763, 32764, 43262] avg=32768.0000 max=65534 max_rank=3 min=7643 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=46 tokens_per_ep_rank=[33098, 32769, 33096, 32096, 32429, 33096, 32775, 32785] avg=32768.0000 max=33098 max_rank=0 min=32096 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=65188 max_load_reduction_pct=0.663248 moved_to_replicas=97274 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] avg=32768.0000 max=33096 max_rank=0 min=32002 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64990 max_load_reduction_pct=0.662582 moved_to_replicas=133002 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=47 tokens_per_ep_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] avg=32768.0000 max=33096 max_rank=0 min=31795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=42604 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=47 tokens_per_ep_rank=[7643, 14643, 32768, 65534, 32767, 32763, 32764, 43262] avg=32768.0000 max=65534 max_rank=3 min=7643 min_rank=0 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=48 tokens_per_ep_rank=[98086, 65515, 2, 4, 2, 68689, 29794, 52] avg=32768.0000 max=98086 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.993347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=47 tokens_per_ep_rank=[33096, 31795, 32768, 33096, 32767, 32763, 32764, 33095] avg=32768.0000 max=33096 max_rank=0 min=31795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=42604 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] avg=32768.0000 max=33096 max_rank=0 min=32002 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64990 max_load_reduction_pct=0.662582 moved_to_replicas=133002 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=48 tokens_per_ep_rank=[98086, 65515, 2, 4, 2, 68689, 29794, 52] avg=32768.0000 max=98086 max_rank=0 min=2 min_rank=2 imbalance_ratio=2.993347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32764, 32227, 33096, 32002, 33096] avg=32768.0000 max=33096 max_rank=0 min=32002 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64990 max_load_reduction_pct=0.662582 moved_to_replicas=133002 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=48 total_expected_recv=1575608 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374528 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=48 total_expected_recv=1568672 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379392 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=48 total_expected_recv=1575495 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374784 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=48 total_expected_recv=1572771 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375936 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=48 total_expected_recv=1567842 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=381440 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=48 total_expected_recv=1575399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375296 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=48 total_expected_recv=1576372 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373120 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=48 total_expected_recv=1570753 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378880 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:47:05] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 723.1 | throughput per GPU (TFLOP/s/GPU): 123.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.810923E+00 | loss scale: 1.0 | grad norm: 3.300 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=31021 expected_overflow=0 exact_recv_per_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=31021 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=31477 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32092 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=31477 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32092 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32259 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=31309 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32259 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31364 expected_overflow=0 exact_recv_per_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=31309 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31364 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32203 expected_overflow=0 exact_recv_per_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32203 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=31815 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32242 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=31326 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32242 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=31326 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=31532 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=30720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=30720 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=31252 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32570 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=31252 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32570 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=30942 expected_overflow=0 exact_recv_per_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=30942 actual_valid_recv=30976 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=31532 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32038 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=30765 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32038 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=30765 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=30976 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=30569 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=30569 actual_valid_recv=30848 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=31433 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=30848 slack_slots=10112 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=31433 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32377 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32377 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=31366 expected_overflow=0 exact_recv_per_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=31366 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=31685 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=31860 expected_overflow=0 exact_recv_per_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=31685 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=31860 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32276 expected_overflow=0 exact_recv_per_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32276 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32406 expected_overflow=0 exact_recv_per_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32406 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=30962 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=30962 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=40960 expected_recv=31988 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=40960 expected_recv=31988 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32168 expected_overflow=0 exact_recv_per_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32168 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32158 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32158 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=40960 expected_recv=31769 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32079 expected_overflow=0 exact_recv_per_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32079 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=40960 expected_recv=31769 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32339 expected_overflow=0 exact_recv_per_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32350 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32350 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32262 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32262 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=40960 expected_recv=31729 expected_overflow=0 exact_recv_per_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32339 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=40960 expected_recv=31729 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=40960 expected_recv=31635 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=40960 expected_recv=31635 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=40960 expected_recv=31659 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=40960 expected_recv=31659 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=40960 expected_recv=31259 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=40960 expected_recv=31259 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32964 expected_overflow=0 exact_recv_per_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32964 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=40960 expected_recv=31970 expected_overflow=0 exact_recv_per_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=40960 expected_recv=31970 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=40960 expected_recv=31769 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=40960 expected_recv=31769 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=40960 expected_recv=31898 expected_overflow=0 exact_recv_per_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=40960 expected_recv=31390 expected_overflow=0 exact_recv_per_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=40960 expected_recv=31898 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=40960 expected_recv=31390 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=40960 expected_recv=31144 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=40960 expected_recv=31714 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=40960 expected_recv=31714 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=40960 expected_recv=31439 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=40960 expected_recv=31439 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=40960 expected_recv=31998 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=40960 expected_recv=31998 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=40960 expected_recv=31144 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32168 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32168 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=40960 expected_recv=31799 expected_overflow=0 exact_recv_per_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=40960 expected_recv=31799 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=40960 expected_recv=31753 expected_overflow=0 exact_recv_per_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=40960 expected_recv=31753 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33068 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33068 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=40960 expected_recv=30802 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=40960 expected_recv=30802 actual_valid_recv=31360 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=40960 expected_recv=31671 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=40960 expected_recv=31671 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32141 expected_overflow=0 exact_recv_per_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32141 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32338 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32338 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33074 expected_overflow=0 exact_recv_per_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33074 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32330 expected_overflow=0 exact_recv_per_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=40960 expected_recv=31152 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32330 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=40960 expected_recv=31152 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=40960 expected_recv=31827 expected_overflow=0 exact_recv_per_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=40960 expected_recv=31827 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32335 expected_overflow=0 exact_recv_per_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32335 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32129 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32129 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32373 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32373 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32272 expected_overflow=0 exact_recv_per_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32272 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32376 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32376 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=24 calls=1 budget=40960 expected_recv=31314 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32373 expected_overflow=0 exact_recv_per_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32373 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=24 calls=1 budget=40960 expected_recv=31164 expected_overflow=0 exact_recv_per_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=24 calls=1 budget=40960 expected_recv=31164 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=26 calls=1 budget=40960 expected_recv=31492 expected_overflow=0 exact_recv_per_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=26 calls=1 budget=40960 expected_recv=31492 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=24 calls=1 budget=40960 expected_recv=31314 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=26 calls=1 budget=40960 expected_recv=31597 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=26 calls=1 budget=40960 expected_recv=31597 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=25 calls=1 budget=40960 expected_recv=31203 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=25 calls=1 budget=40960 expected_recv=31203 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=27 calls=1 budget=40960 expected_recv=31857 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=27 calls=1 budget=40960 expected_recv=31857 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32131 expected_overflow=0 exact_recv_per_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=28 calls=1 budget=40960 expected_recv=31372 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32131 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=28 calls=1 budget=40960 expected_recv=31372 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32474 expected_overflow=0 exact_recv_per_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=28 calls=1 budget=40960 expected_recv=31117 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32474 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=28 calls=1 budget=40960 expected_recv=31117 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=29 calls=1 budget=40960 expected_recv=31819 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=30 calls=1 budget=40960 expected_recv=31792 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=29 calls=1 budget=40960 expected_recv=31819 actual_valid_recv=32256 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=30 calls=1 budget=40960 expected_recv=31792 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32123 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32123 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32136 expected_overflow=0 exact_recv_per_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32454 expected_overflow=0 exact_recv_per_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32136 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32454 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32107 expected_overflow=0 exact_recv_per_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32107 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=34 calls=1 budget=40960 expected_recv=31733 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=34 calls=1 budget=40960 expected_recv=31733 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=33 calls=1 budget=40960 expected_recv=31157 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=33 calls=1 budget=40960 expected_recv=31157 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=33 calls=1 budget=40960 expected_recv=31815 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=33 calls=1 budget=40960 expected_recv=31815 actual_valid_recv=32384 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32131 expected_overflow=0 exact_recv_per_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32131 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32051 expected_overflow=0 exact_recv_per_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32051 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32445 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32445 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=35 calls=1 budget=40960 expected_recv=31735 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32118 expected_overflow=0 exact_recv_per_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=37 calls=1 budget=40960 expected_recv=31487 expected_overflow=0 exact_recv_per_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32155 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=35 calls=1 budget=40960 expected_recv=31735 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32155 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32118 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=37 calls=1 budget=40960 expected_recv=31487 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32474 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32474 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32090 expected_overflow=0 exact_recv_per_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32090 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=38 calls=1 budget=40960 expected_recv=31942 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=38 calls=1 budget=40960 expected_recv=31942 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=38 calls=1 budget=40960 expected_recv=31520 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=38 calls=1 budget=40960 expected_recv=31520 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=39 calls=1 budget=40960 expected_recv=31748 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=39 calls=1 budget=40960 expected_recv=31748 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=39 calls=1 budget=40960 expected_recv=31470 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=39 calls=1 budget=40960 expected_recv=31470 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=40 calls=1 budget=40960 expected_recv=31293 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33079 expected_overflow=0 exact_recv_per_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33079 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=40 calls=1 budget=40960 expected_recv=31293 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=41 calls=1 budget=40960 expected_recv=31805 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=41 calls=1 budget=40960 expected_recv=31805 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=40 calls=1 budget=40960 expected_recv=31483 expected_overflow=0 exact_recv_per_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=40 calls=1 budget=40960 expected_recv=31483 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=42 calls=1 budget=40960 expected_recv=31816 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=43 calls=1 budget=40960 expected_recv=31439 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=42 calls=1 budget=40960 expected_recv=31816 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=43 calls=1 budget=40960 expected_recv=31439 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32404 expected_overflow=0 exact_recv_per_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32404 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32203 expected_overflow=0 exact_recv_per_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32242 expected_overflow=0 exact_recv_per_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32203 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32242 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=46 calls=1 budget=40960 expected_recv=31784 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=46 calls=1 budget=40960 expected_recv=31784 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=43 calls=1 budget=40960 expected_recv=31635 expected_overflow=0 exact_recv_per_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33044 expected_overflow=0 exact_recv_per_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=43 calls=1 budget=40960 expected_recv=31635 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33044 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32370 expected_overflow=0 exact_recv_per_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32370 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=45 calls=1 budget=40960 expected_recv=31817 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=45 calls=1 budget=40960 expected_recv=31817 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32045 expected_overflow=0 exact_recv_per_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32045 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32517 expected_overflow=0 exact_recv_per_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32517 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32195 expected_overflow=0 exact_recv_per_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32195 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[23222, 27900, 32547, 45668, 42289, 24464, 26844, 39210] avg=32768.0000 max=45668 max_rank=3 min=23222 min_rank=0 imbalance_ratio=1.393677 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=47 calls=1 budget=40960 expected_recv=31254 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=47 calls=1 budget=40960 expected_recv=31254 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31021 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12571 max_load_reduction_pct=0.275269 moved_to_replicas=27879 active_clone_links=8 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[23222, 27900, 32547, 45668, 42289, 24464, 26844, 39210] avg=32768.0000 max=45668 max_rank=3 min=23222 min_rank=0 imbalance_ratio=1.393677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[23222, 27900, 32547, 45668, 42289, 24464, 26844, 39210] avg=32768.0000 max=45668 max_rank=3 min=23222 min_rank=0 imbalance_ratio=1.393677 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[23222, 27900, 32547, 45668, 42289, 24464, 26844, 39210] avg=32768.0000 max=45668 max_rank=3 min=23222 min_rank=0 imbalance_ratio=1.393677 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[23222, 27900, 32547, 45668, 42289, 24464, 26844, 39210] avg=32768.0000 max=45668 max_rank=3 min=23222 min_rank=0 imbalance_ratio=1.393677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[64257, 87, 3532, 35048, 28654, 72506, 32092, 25968] avg=32768.0000 max=72506 max_rank=5 min=87 min_rank=1 imbalance_ratio=2.212708 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31021 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12571 max_load_reduction_pct=0.275269 moved_to_replicas=27879 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31021 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12571 max_load_reduction_pct=0.275269 moved_to_replicas=27879 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[23222, 27900, 32547, 45668, 42289, 24464, 26844, 39210] avg=32768.0000 max=45668 max_rank=3 min=23222 min_rank=0 imbalance_ratio=1.393677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31021 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12571 max_load_reduction_pct=0.275269 moved_to_replicas=27879 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31021 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12571 max_load_reduction_pct=0.275269 moved_to_replicas=27879 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] avg=32768.0000 max=33097 max_rank=2 min=31477 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39409 max_load_reduction_pct=0.543527 moved_to_replicas=72523 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[64257, 87, 3532, 35048, 28654, 72506, 32092, 25968] avg=32768.0000 max=72506 max_rank=5 min=87 min_rank=1 imbalance_ratio=2.212708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[64257, 87, 3532, 35048, 28654, 72506, 32092, 25968] avg=32768.0000 max=72506 max_rank=5 min=87 min_rank=1 imbalance_ratio=2.212708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31021 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12571 max_load_reduction_pct=0.275269 moved_to_replicas=27879 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[23258, 24436, 36597, 41616, 42832, 25848, 30668, 36889] avg=32768.0000 max=42832 max_rank=4 min=23258 min_rank=0 imbalance_ratio=1.307129 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32994, 32264, 33097, 33095, 33096, 31407, 33095, 33096] avg=32768.0000 max=33097 max_rank=2 min=31407 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=9735 max_load_reduction_pct=0.227283 moved_to_replicas=25550 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[54608, 2087, 12278, 36155, 43990, 63047, 37200, 12779] avg=32768.0000 max=63047 max_rank=5 min=2087 min_rank=1 imbalance_ratio=1.924042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33096, 30472, 33094, 33097, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=29950 max_load_reduction_pct=0.475042 moved_to_replicas=69520 active_clone_links=9 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[68188, 13897, 36714, 8523, 50222, 36106, 22707, 25787] avg=32768.0000 max=68188 max_rank=0 min=8523 min_rank=3 imbalance_ratio=2.080933 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[33096, 31151, 33096, 33097, 33096, 33096, 33097, 32415] avg=32768.0000 max=33097 max_rank=3 min=31151 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35091 max_load_reduction_pct=0.514621 moved_to_replicas=58846 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[2509, 14011, 33923, 57011, 24860, 31815, 17593, 80422] avg=32768.0000 max=80422 max_rank=7 min=2509 min_rank=0 imbalance_ratio=2.454285 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[33096, 31752, 33096, 33095, 33096, 31815, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31752 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=47325 max_load_reduction_pct=0.588458 moved_to_replicas=72068 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65193, 21760, 52231, 62201, 58096, 145, 1581, 937] avg=32768.0000 max=65193 max_rank=0 min=145 min_rank=5 imbalance_ratio=1.989532 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32242, 33095, 31326] avg=32768.0000 max=33097 max_rank=4 min=31326 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32096 max_load_reduction_pct=0.492323 moved_to_replicas=105337 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[58516, 4326, 33935, 45613, 50282, 37886, 31199, 387] avg=32768.0000 max=58516 max_rank=0 min=387 min_rank=7 imbalance_ratio=1.785767 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33094, 32038, 31532] avg=32768.0000 max=33096 max_rank=0 min=31532 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25420 max_load_reduction_pct=0.434411 moved_to_replicas=60752 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32803, 221, 18322, 50496, 100, 41145, 58443, 60614] avg=32768.0000 max=60614 max_rank=7 min=100 min_rank=4 imbalance_ratio=1.849792 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32803, 30765, 33096, 33096, 33098, 33094, 33096, 33096] avg=32768.0000 max=33098 max_rank=4 min=30765 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=27516 max_load_reduction_pct=0.453955 moved_to_replicas=78314 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[44964, 41, 48641, 27922, 32999, 54185, 581, 52811] avg=32768.0000 max=54185 max_rank=5 min=41 min_rank=1 imbalance_ratio=1.653595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33097, 33095, 33096, 30569, 32999, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30569 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=21088 max_load_reduction_pct=0.389185 moved_to_replicas=68217 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[6391, 94389, 32377, 358, 32854, 39193, 50748, 5834] avg=32768.0000 max=94389 max_rank=1 min=358 min_rank=3 imbalance_ratio=2.880524 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[31433, 33096, 32377, 33096, 32854, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61293 max_load_reduction_pct=0.649366 moved_to_replicas=85042 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[1970, 33130, 66750, 30731, 22442, 25449, 32780, 48892] avg=32768.0000 max=66750 max_rank=2 min=1970 min_rank=0 imbalance_ratio=2.037048 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 32922, 30962, 33096, 32780, 33096] avg=32768.0000 max=33096 max_rank=0 min=30962 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33654 max_load_reduction_pct=0.504180 moved_to_replicas=49484 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[35620, 41311, 33271, 46991, 50962, 32519, 21419, 51] avg=32768.0000 max=50962 max_rank=4 min=51 min_rank=7 imbalance_ratio=1.555237 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33096, 32519, 32158, 31988] avg=32768.0000 max=33096 max_rank=0 min=31988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=17866 max_load_reduction_pct=0.350575 moved_to_replicas=42675 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[16377, 52877, 35636, 32545, 29810, 34874, 59946, 79] avg=32768.0000 max=59946 max_rank=6 min=79 min_rank=7 imbalance_ratio=1.829407 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[33096, 33096, 33096, 32545, 32350, 33096, 33096, 31769] avg=32768.0000 max=33096 max_rank=0 min=31769 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=26850 max_load_reduction_pct=0.447903 moved_to_replicas=50949 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[35776, 33186, 70885, 32262, 19, 40654, 59, 49303] avg=32768.0000 max=70885 max_rank=2 min=19 min_rank=4 imbalance_ratio=2.163239 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 32262, 32768, 33096, 31635, 33095] avg=32768.0000 max=33096 max_rank=0 min=31635 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37789 max_load_reduction_pct=0.533103 moved_to_replicas=64324 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32691, 98188, 28371, 33406, 3765, 288, 30, 65405] avg=32768.0000 max=98188 max_rank=1 min=30 min_rank=6 imbalance_ratio=2.996460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32691, 33096, 31659, 33096, 33096, 32631, 32779, 33096] avg=32768.0000 max=33096 max_rank=1 min=31659 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65092 max_load_reduction_pct=0.662932 moved_to_replicas=97711 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32793, 41370, 4360, 20, 79894, 32951, 70738, 18] avg=32768.0000 max=79894 max_rank=4 min=18 min_rank=7 imbalance_ratio=2.438171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32793, 33097, 33096, 31259, 33095, 32951, 33095, 32758] avg=32768.0000 max=33097 max_rank=1 min=31259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46797 max_load_reduction_pct=0.585739 moved_to_replicas=92714 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[11309, 42649, 65455, 32751, 32535, 65579, 220, 11646] avg=32768.0000 max=65579 max_rank=5 min=220 min_rank=6 imbalance_ratio=2.001312 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[33096, 33098, 33096, 32751, 32535, 33096, 32703, 31769] avg=32768.0000 max=33098 max_rank=1 min=31769 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32481 max_load_reduction_pct=0.495296 moved_to_replicas=74395 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[18, 54, 1853, 98201, 16231, 100907, 32589, 12291] avg=32768.0000 max=100907 max_rank=5 min=18 min_rank=0 imbalance_ratio=3.079437 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[32767, 32690, 33096, 33096, 31714, 33096, 32589, 33096] avg=32768.0000 max=33096 max_rank=2 min=31714 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67811 max_load_reduction_pct=0.672015 moved_to_replicas=132916 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[51519, 25698, 10099, 32771, 61373, 32855, 32695, 15134] avg=32768.0000 max=61373 max_rank=4 min=10099 min_rank=2 imbalance_ratio=1.872955 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[33096, 31439, 33097, 32771, 33097, 32855, 32695, 33094] avg=32768.0000 max=33097 max_rank=2 min=31439 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=28276 max_load_reduction_pct=0.460724 moved_to_replicas=46700 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[30926, 0, 32776, 52434, 32781, 10592, 32567, 70068] avg=32768.0000 max=70068 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.138306 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[31998, 32734, 32776, 33096, 32781, 33095, 32567, 33097] avg=32768.0000 max=33097 max_rank=7 min=31998 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=36971 max_load_reduction_pct=0.527645 moved_to_replicas=56310 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[32794, 58238, 33068, 49711, 299, 17819, 36011, 34204] avg=32768.0000 max=58238 max_rank=1 min=299 min_rank=4 imbalance_ratio=1.777283 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[32794, 33096, 33068, 33096, 30802, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30802 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=25142 max_load_reduction_pct=0.431711 moved_to_replicas=45780 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[5642, 32628, 35, 47, 72493, 57722, 60894, 32683] avg=32768.0000 max=72493 max_rank=4 min=35 min_rank=2 imbalance_ratio=2.212311 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[33096, 32628, 32778, 31671, 33096, 33096, 33096, 32683] avg=32768.0000 max=33096 max_rank=0 min=31671 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39397 max_load_reduction_pct=0.543459 moved_to_replicas=91821 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[98235, 36, 54, 22, 65588, 32774, 44529, 20906] avg=32768.0000 max=98235 max_rank=0 min=22 min_rank=3 imbalance_ratio=2.997894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[33096, 32426, 32546, 32771, 33096, 32774, 33097, 32338] avg=32768.0000 max=33097 max_rank=6 min=32338 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65138 max_load_reduction_pct=0.663083 moved_to_replicas=109064 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[32743, 65518, 65524, 26059, 52942, 9, 29, 19320] avg=32768.0000 max=65524 max_rank=2 min=9 min_rank=5 imbalance_ratio=1.999634 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[32743, 33096, 33096, 32129, 33097, 32437, 32451, 33095] avg=32768.0000 max=33097 max_rank=4 min=32129 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=84696 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[32675, 186, 10310, 32741, 32440, 65358, 65536, 22898] avg=32768.0000 max=65536 max_rank=6 min=186 min_rank=1 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[32675, 32626, 33097, 32741, 32440, 33096, 33096, 32373] avg=32768.0000 max=33097 max_rank=2 min=32373 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=64702 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=25 tokens_per_ep_rank=[5060, 91546, 67689, 64881, 0, 18, 32756, 194] avg=32768.0000 max=91546 max_rank=1 min=0 min_rank=4 imbalance_ratio=2.793762 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=25 tokens_per_ep_rank=[33094, 33096, 33096, 33096, 32749, 31314, 32756, 32943] avg=32768.0000 max=33096 max_rank=1 min=31314 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=58450 max_load_reduction_pct=0.638477 moved_to_replicas=124828 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=26 tokens_per_ep_rank=[110, 32775, 32735, 32875, 65362, 49, 65488, 32750] avg=32768.0000 max=65488 max_rank=6 min=49 min_rank=5 imbalance_ratio=1.998535 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=26 tokens_per_ep_rank=[32376, 32775, 32735, 32875, 33096, 32441, 33096, 32750] avg=32768.0000 max=33096 max_rank=4 min=32376 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=64658 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=27 tokens_per_ep_rank=[28076, 32751, 68178, 2, 98448, 1436, 33195, 58] avg=32768.0000 max=98448 max_rank=4 min=2 min_rank=3 imbalance_ratio=3.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=27 tokens_per_ep_rank=[31597, 32751, 33096, 32748, 33096, 33096, 33096, 32664] avg=32768.0000 max=33096 max_rank=2 min=31597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65352 max_load_reduction_pct=0.663823 moved_to_replicas=100533 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=28 tokens_per_ep_rank=[0, 32770, 18, 130315, 66237, 32750, 18, 36] avg=32768.0000 max=130315 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.976898 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=28 tokens_per_ep_rank=[32749, 32770, 32767, 33096, 33096, 32750, 32131, 32785] avg=32768.0000 max=33096 max_rank=3 min=32131 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97219 max_load_reduction_pct=0.746031 moved_to_replicas=130360 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=29 tokens_per_ep_rank=[62691, 1417, 65522, 42326, 53100, 32617, 249, 4222] avg=32768.0000 max=65522 max_rank=2 min=249 min_rank=6 imbalance_ratio=1.999573 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=29 tokens_per_ep_rank=[33096, 31372, 33096, 33096, 33097, 32617, 32675, 33095] avg=32768.0000 max=33097 max_rank=4 min=31372 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32425 max_load_reduction_pct=0.494872 moved_to_replicas=91255 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=30 tokens_per_ep_rank=[32783, 80591, 0, 32773, 32721, 19, 50454, 32803] avg=32768.0000 max=80591 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.459442 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=30 tokens_per_ep_rank=[32783, 33096, 32748, 32773, 32721, 32123, 33097, 32803] avg=32768.0000 max=33097 max_rank=6 min=32123 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47494 max_load_reduction_pct=0.589321 moved_to_replicas=64853 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=31 tokens_per_ep_rank=[54, 71, 66370, 26908, 38408, 32750, 0, 97583] avg=32768.0000 max=97583 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.977997 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=31 tokens_per_ep_rank=[31792, 32820, 33096, 32746, 33095, 32750, 32749, 33096] avg=32768.0000 max=33096 max_rank=2 min=31792 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64487 max_load_reduction_pct=0.660843 moved_to_replicas=103073 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=32 tokens_per_ep_rank=[22, 32769, 7390, 65519, 32786, 64100, 26808, 32750] avg=32768.0000 max=65519 max_rank=3 min=22 min_rank=0 imbalance_ratio=1.999481 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=32 tokens_per_ep_rank=[32445, 32769, 33095, 33096, 32786, 33096, 32107, 32750] avg=32768.0000 max=33096 max_rank=3 min=32107 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=63427 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=33 tokens_per_ep_rank=[18, 32602, 65511, 18, 32761, 61, 98303, 32870] avg=32768.0000 max=98303 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.999969 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=33 tokens_per_ep_rank=[32767, 32602, 33096, 32476, 32761, 32476, 33096, 32870] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65207 max_load_reduction_pct=0.663327 moved_to_replicas=97622 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=34 tokens_per_ep_rank=[32758, 69443, 94316, 18, 50, 22, 0, 65537] avg=32768.0000 max=94316 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.878296 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=34 tokens_per_ep_rank=[32758, 33097, 33098, 33094, 33095, 31157, 32749, 33096] avg=32768.0000 max=33098 max_rank=2 min=31157 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=61218 max_load_reduction_pct=0.649073 moved_to_replicas=130008 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=35 tokens_per_ep_rank=[2081, 296, 32804, 126588, 0, 2864, 64984, 32527] avg=32768.0000 max=126588 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.863159 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=35 tokens_per_ep_rank=[31733, 33045, 32804, 33094, 32749, 33096, 33096, 32527] avg=32768.0000 max=33096 max_rank=5 min=31733 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93492 max_load_reduction_pct=0.738553 moved_to_replicas=125380 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=36 tokens_per_ep_rank=[65455, 31735, 22719, 20603, 18, 65552, 32768, 23294] avg=32768.0000 max=65552 max_rank=5 min=18 min_rank=4 imbalance_ratio=2.000488 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=36 tokens_per_ep_rank=[33096, 31735, 33097, 33095, 32474, 33096, 32768, 32783] avg=32768.0000 max=33097 max_rank=2 min=31735 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32455 max_load_reduction_pct=0.495103 moved_to_replicas=64815 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=37 tokens_per_ep_rank=[9, 32786, 32795, 32694, 57977, 0, 73109, 32774] avg=32768.0000 max=73109 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.231110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=37 tokens_per_ep_rank=[32155, 32786, 32795, 32694, 33095, 32749, 33096, 32774] avg=32768.0000 max=33096 max_rank=6 min=32155 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=40013 max_load_reduction_pct=0.547306 moved_to_replicas=64894 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=38 tokens_per_ep_rank=[32768, 0, 98268, 32764, 66, 32749, 19, 65510] avg=32768.0000 max=98268 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.998901 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=38 tokens_per_ep_rank=[32768, 32749, 33096, 32764, 32480, 32749, 32442, 33096] avg=32768.0000 max=33096 max_rank=2 min=32442 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=97586 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=39 tokens_per_ep_rank=[0, 32767, 65549, 8844, 204, 36945, 86315, 31520] avg=32768.0000 max=86315 max_rank=6 min=0 min_rank=0 imbalance_ratio=2.634125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=39 tokens_per_ep_rank=[32749, 32767, 33096, 33097, 32722, 33096, 33097, 31520] avg=32768.0000 max=33097 max_rank=3 min=31520 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=53218 max_load_reduction_pct=0.616556 moved_to_replicas=89521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=40 tokens_per_ep_rank=[9404, 65524, 3933, 65526, 31748, 53237, 32772, 0] avg=32768.0000 max=65526 max_rank=3 min=0 min_rank=7 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=40 tokens_per_ep_rank=[32809, 33096, 33096, 33096, 31748, 33097, 32772, 32430] avg=32768.0000 max=33097 max_rank=5 min=31748 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32429 max_load_reduction_pct=0.494903 moved_to_replicas=84999 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=41 tokens_per_ep_rank=[36380, 18, 93726, 26598, 33, 39645, 242, 65502] avg=32768.0000 max=93726 max_rank=2 min=18 min_rank=1 imbalance_ratio=2.860291 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=41 tokens_per_ep_rank=[33096, 32723, 33096, 33095, 31293, 33097, 32648, 33096] avg=32768.0000 max=33097 max_rank=5 min=31293 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=60629 max_load_reduction_pct=0.646875 moved_to_replicas=102869 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=42 tokens_per_ep_rank=[32721, 43712, 20268, 65543, 65527, 32767, 20, 1586] avg=32768.0000 max=65543 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.000214 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=42 tokens_per_ep_rank=[32721, 33096, 31805, 33096, 33096, 32767, 32467, 33096] avg=32768.0000 max=33096 max_rank=1 min=31805 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=75494 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=43 tokens_per_ep_rank=[72670, 26568, 31816, 36, 32786, 9, 65491, 32768] avg=32768.0000 max=72670 max_rank=0 min=9 min_rank=5 imbalance_ratio=2.217712 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32728, 31816, 33096, 32786, 32758, 33096, 32768] avg=32768.0000 max=33096 max_rank=0 min=31816 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39574 max_load_reduction_pct=0.544571 moved_to_replicas=71969 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=44 tokens_per_ep_rank=[32759, 36, 39662, 58926, 724, 32778, 97216, 43] avg=32768.0000 max=97216 max_rank=6 min=36 min_rank=1 imbalance_ratio=2.966797 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=44 tokens_per_ep_rank=[32759, 32784, 33095, 33096, 33097, 32778, 33096, 31439] avg=32768.0000 max=33097 max_rank=4 min=31439 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64119 max_load_reduction_pct=0.659552 moved_to_replicas=96516 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 65494, 24, 32791] avg=32768.0000 max=65494 max_rank=5 min=24 min_rank=6 imbalance_ratio=1.998718 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=45 tokens_per_ep_rank=[32781, 32750, 32768, 32757, 32779, 33096, 32422, 32791] avg=32768.0000 max=33096 max_rank=5 min=32422 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32398 max_load_reduction_pct=0.494671 moved_to_replicas=32398 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=46 tokens_per_ep_rank=[8117, 45, 36, 62939, 32767, 23670, 134569, 1] avg=32768.0000 max=134569 max_rank=6 min=1 min_rank=7 imbalance_ratio=4.106720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=46 tokens_per_ep_rank=[33096, 32739, 32783, 33096, 32767, 31817, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31817 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=101473 max_load_reduction_pct=0.754059 moved_to_replicas=131316 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=47 tokens_per_ep_rank=[40, 28452, 32768, 96868, 32758, 32793, 38461, 4] avg=32768.0000 max=96868 max_rank=3 min=4 min_rank=7 imbalance_ratio=2.956177 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=47 tokens_per_ep_rank=[31784, 33096, 32768, 33096, 32758, 32793, 33096, 32753] avg=32768.0000 max=33096 max_rank=1 min=31784 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63772 max_load_reduction_pct=0.658339 moved_to_replicas=69137 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 65513, 22, 36, 65609, 32737, 32822] avg=32768.0000 max=65609 max_rank=5 min=22 min_rank=3 imbalance_ratio=2.002228 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=48 tokens_per_ep_rank=[32655, 32750, 33096, 32535, 32453, 33096, 32737, 32822] avg=32768.0000 max=33096 max_rank=2 min=32453 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32513 max_load_reduction_pct=0.495557 moved_to_replicas=64930 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=48 total_expected_recv=1571791 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376448 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=48 total_expected_recv=1571972 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376832 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=48 total_expected_recv=1569790 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377600 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=48 total_expected_recv=1575441 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372352 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=48 total_expected_recv=1580067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368000 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=48 total_expected_recv=1567189 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380672 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=48 total_expected_recv=1571442 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376320 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=48 total_expected_recv=1575220 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374016 peak_demand=33097 required_capacity=1.010040 - [2026-08-26 15:01:27] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 724.3 | throughput per GPU (TFLOP/s/GPU): 123.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.773178E+00 | loss scale: 1.0 | grad norm: 2.826 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32346 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32112 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32206 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32346 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32112 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[64257, 87, 3532, 35048, 28654, 72506, 32092, 25968] avg=32768.0000 max=72506 max_rank=5 min=87 min_rank=1 imbalance_ratio=2.212708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[90320, 21779, 9390, 5765, 28875, 48362, 21173, 36480] avg=32768.0000 max=90320 max_rank=0 min=5765 min_rank=3 imbalance_ratio=2.756348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[64257, 87, 3532, 35048, 28654, 72506, 32092, 25968] avg=32768.0000 max=72506 max_rank=5 min=87 min_rank=1 imbalance_ratio=2.212708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] avg=32768.0000 max=33097 max_rank=2 min=31477 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39409 max_load_reduction_pct=0.543527 moved_to_replicas=72523 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[23222, 27900, 32547, 45668, 42289, 24464, 26844, 39210] avg=32768.0000 max=45668 max_rank=3 min=23222 min_rank=0 imbalance_ratio=1.393677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] avg=32768.0000 max=33097 max_rank=2 min=31477 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39409 max_load_reduction_pct=0.543527 moved_to_replicas=72523 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[64257, 87, 3532, 35048, 28654, 72506, 32092, 25968] avg=32768.0000 max=72506 max_rank=5 min=87 min_rank=1 imbalance_ratio=2.212708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] avg=32768.0000 max=33097 max_rank=2 min=31477 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39409 max_load_reduction_pct=0.543527 moved_to_replicas=72523 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31309 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57224 max_load_reduction_pct=0.633570 moved_to_replicas=75874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] avg=32768.0000 max=33097 max_rank=2 min=31477 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39409 max_load_reduction_pct=0.543527 moved_to_replicas=72523 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[23222, 27900, 32547, 45668, 42289, 24464, 26844, 39210] avg=32768.0000 max=45668 max_rank=3 min=23222 min_rank=0 imbalance_ratio=1.393677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31021 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12571 max_load_reduction_pct=0.275269 moved_to_replicas=27879 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[90320, 21779, 9390, 5765, 28875, 48362, 21173, 36480] avg=32768.0000 max=90320 max_rank=0 min=5765 min_rank=3 imbalance_ratio=2.756348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] avg=32768.0000 max=33097 max_rank=2 min=31477 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39409 max_load_reduction_pct=0.543527 moved_to_replicas=72523 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[90320, 21779, 9390, 5765, 28875, 48362, 21173, 36480] avg=32768.0000 max=90320 max_rank=0 min=5765 min_rank=3 imbalance_ratio=2.756348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[90320, 21779, 9390, 5765, 28875, 48362, 21173, 36480] avg=32768.0000 max=90320 max_rank=0 min=5765 min_rank=3 imbalance_ratio=2.756348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31021, 32547, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31021 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=12571 max_load_reduction_pct=0.275269 moved_to_replicas=27879 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[4272, 14, 35980, 11484, 19701, 49743, 66960, 73990] avg=32768.0000 max=73990 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.257996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31309 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57224 max_load_reduction_pct=0.633570 moved_to_replicas=75874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[90320, 21779, 9390, 5765, 28875, 48362, 21173, 36480] avg=32768.0000 max=90320 max_rank=0 min=5765 min_rank=3 imbalance_ratio=2.756348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31309 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57224 max_load_reduction_pct=0.633570 moved_to_replicas=75874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[64257, 87, 3532, 35048, 28654, 72506, 32092, 25968] avg=32768.0000 max=72506 max_rank=5 min=87 min_rank=1 imbalance_ratio=2.212708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31309 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57224 max_load_reduction_pct=0.633570 moved_to_replicas=75874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[90320, 21779, 9390, 5765, 28875, 48362, 21173, 36480] avg=32768.0000 max=90320 max_rank=0 min=5765 min_rank=3 imbalance_ratio=2.756348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31364 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40893 max_load_reduction_pct=0.552683 moved_to_replicas=94289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31309 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57224 max_load_reduction_pct=0.633570 moved_to_replicas=75874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[64257, 87, 3532, 35048, 28654, 72506, 32092, 25968] avg=32768.0000 max=72506 max_rank=5 min=87 min_rank=1 imbalance_ratio=2.212708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] avg=32768.0000 max=33097 max_rank=2 min=31477 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39409 max_load_reduction_pct=0.543527 moved_to_replicas=72523 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[4272, 14, 35980, 11484, 19701, 49743, 66960, 73990] avg=32768.0000 max=73990 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.257996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31309 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57224 max_load_reduction_pct=0.633570 moved_to_replicas=75874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[4272, 14, 35980, 11484, 19701, 49743, 66960, 73990] avg=32768.0000 max=73990 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.257996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[4272, 14, 35980, 11484, 19701, 49743, 66960, 73990] avg=32768.0000 max=73990 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.257996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[51923, 29214, 47246, 52161, 39790, 7676, 19, 34115] avg=32768.0000 max=52161 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.591827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33095, 33097, 33095, 33097, 33096, 32092, 31477] avg=32768.0000 max=33097 max_rank=2 min=31477 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39409 max_load_reduction_pct=0.543527 moved_to_replicas=72523 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31364 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40893 max_load_reduction_pct=0.552683 moved_to_replicas=94289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[4272, 14, 35980, 11484, 19701, 49743, 66960, 73990] avg=32768.0000 max=73990 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.257996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31364 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40893 max_load_reduction_pct=0.552683 moved_to_replicas=94289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[90320, 21779, 9390, 5765, 28875, 48362, 21173, 36480] avg=32768.0000 max=90320 max_rank=0 min=5765 min_rank=3 imbalance_ratio=2.756348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31364 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40893 max_load_reduction_pct=0.552683 moved_to_replicas=94289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[4272, 14, 35980, 11484, 19701, 49743, 66960, 73990] avg=32768.0000 max=73990 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.257996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=19064 max_load_reduction_pct=0.365484 moved_to_replicas=59755 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31364 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40893 max_load_reduction_pct=0.552683 moved_to_replicas=94289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[90320, 21779, 9390, 5765, 28875, 48362, 21173, 36480] avg=32768.0000 max=90320 max_rank=0 min=5765 min_rank=3 imbalance_ratio=2.756348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[51923, 29214, 47246, 52161, 39790, 7676, 19, 34115] avg=32768.0000 max=52161 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.591827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31309 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57224 max_load_reduction_pct=0.633570 moved_to_replicas=75874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31364 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40893 max_load_reduction_pct=0.552683 moved_to_replicas=94289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[51923, 29214, 47246, 52161, 39790, 7676, 19, 34115] avg=32768.0000 max=52161 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.591827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[51923, 29214, 47246, 52161, 39790, 7676, 19, 34115] avg=32768.0000 max=52161 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.591827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[92851, 2521, 40938, 19654, 2364, 2229, 32570, 69017] avg=32768.0000 max=92851 max_rank=0 min=2229 min_rank=5 imbalance_ratio=2.833588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33096, 31309, 33096, 32259, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31309 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57224 max_load_reduction_pct=0.633570 moved_to_replicas=75874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=19064 max_load_reduction_pct=0.365484 moved_to_replicas=59755 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[51923, 29214, 47246, 52161, 39790, 7676, 19, 34115] avg=32768.0000 max=52161 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.591827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=19064 max_load_reduction_pct=0.365484 moved_to_replicas=59755 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=19064 max_load_reduction_pct=0.365484 moved_to_replicas=59755 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[4272, 14, 35980, 11484, 19701, 49743, 66960, 73990] avg=32768.0000 max=73990 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.257996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[51923, 29214, 47246, 52161, 39790, 7676, 19, 34115] avg=32768.0000 max=52161 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.591827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] avg=32768.0000 max=33097 max_rank=2 min=31252 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=59754 max_load_reduction_pct=0.643547 moved_to_replicas=103518 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=19064 max_load_reduction_pct=0.365484 moved_to_replicas=59755 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[4272, 14, 35980, 11484, 19701, 49743, 66960, 73990] avg=32768.0000 max=73990 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.257996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31364 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40893 max_load_reduction_pct=0.552683 moved_to_replicas=94289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[92851, 2521, 40938, 19654, 2364, 2229, 32570, 69017] avg=32768.0000 max=92851 max_rank=0 min=2229 min_rank=5 imbalance_ratio=2.833588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[92851, 2521, 40938, 19654, 2364, 2229, 32570, 69017] avg=32768.0000 max=92851 max_rank=0 min=2229 min_rank=5 imbalance_ratio=2.833588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=19064 max_load_reduction_pct=0.365484 moved_to_replicas=59755 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[92851, 2521, 40938, 19654, 2364, 2229, 32570, 69017] avg=32768.0000 max=92851 max_rank=0 min=2229 min_rank=5 imbalance_ratio=2.833588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[0, 51029, 62406, 27616, 32836, 32886, 1091, 54280] avg=32768.0000 max=62406 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.904480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[31364, 32203, 33096, 33096, 33095, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31364 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40893 max_load_reduction_pct=0.552683 moved_to_replicas=94289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[92851, 2521, 40938, 19654, 2364, 2229, 32570, 69017] avg=32768.0000 max=92851 max_rank=0 min=2229 min_rank=5 imbalance_ratio=2.833588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] avg=32768.0000 max=33097 max_rank=2 min=31252 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=59754 max_load_reduction_pct=0.643547 moved_to_replicas=103518 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] avg=32768.0000 max=33097 max_rank=2 min=31252 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=59754 max_load_reduction_pct=0.643547 moved_to_replicas=103518 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[51923, 29214, 47246, 52161, 39790, 7676, 19, 34115] avg=32768.0000 max=52161 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.591827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] avg=32768.0000 max=33097 max_rank=2 min=31252 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=59754 max_load_reduction_pct=0.643547 moved_to_replicas=103518 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=30942 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=29308 max_load_reduction_pct=0.469634 moved_to_replicas=68427 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[92851, 2521, 40938, 19654, 2364, 2229, 32570, 69017] avg=32768.0000 max=92851 max_rank=0 min=2229 min_rank=5 imbalance_ratio=2.833588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] avg=32768.0000 max=33097 max_rank=2 min=31252 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=59754 max_load_reduction_pct=0.643547 moved_to_replicas=103518 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[51923, 29214, 47246, 52161, 39790, 7676, 19, 34115] avg=32768.0000 max=52161 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.591827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=19064 max_load_reduction_pct=0.365484 moved_to_replicas=59755 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] avg=32768.0000 max=33097 max_rank=2 min=31252 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=59754 max_load_reduction_pct=0.643547 moved_to_replicas=103518 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[0, 51029, 62406, 27616, 32836, 32886, 1091, 54280] avg=32768.0000 max=62406 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.904480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[0, 51029, 62406, 27616, 32836, 32886, 1091, 54280] avg=32768.0000 max=62406 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.904480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[0, 51029, 62406, 27616, 32836, 32886, 1091, 54280] avg=32768.0000 max=62406 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.904480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[67696, 32757, 63545, 11564, 32888, 52548, 1116, 30] avg=32768.0000 max=67696 max_rank=0 min=30 min_rank=7 imbalance_ratio=2.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[33095, 30472, 33096, 33096, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=19064 max_load_reduction_pct=0.365484 moved_to_replicas=59755 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[0, 51029, 62406, 27616, 32836, 32886, 1091, 54280] avg=32768.0000 max=62406 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.904480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=30942 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=29308 max_load_reduction_pct=0.469634 moved_to_replicas=68427 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=30942 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=29308 max_load_reduction_pct=0.469634 moved_to_replicas=68427 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=30942 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=29308 max_load_reduction_pct=0.469634 moved_to_replicas=68427 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31366 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34600 max_load_reduction_pct=0.511108 moved_to_replicas=84501 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[0, 51029, 62406, 27616, 32836, 32886, 1091, 54280] avg=32768.0000 max=62406 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.904480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[92851, 2521, 40938, 19654, 2364, 2229, 32570, 69017] avg=32768.0000 max=92851 max_rank=0 min=2229 min_rank=5 imbalance_ratio=2.833588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=30942 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=29308 max_load_reduction_pct=0.469634 moved_to_replicas=68427 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[92851, 2521, 40938, 19654, 2364, 2229, 32570, 69017] avg=32768.0000 max=92851 max_rank=0 min=2229 min_rank=5 imbalance_ratio=2.833588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[67696, 32757, 63545, 11564, 32888, 52548, 1116, 30] avg=32768.0000 max=67696 max_rank=0 min=30 min_rank=7 imbalance_ratio=2.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[67696, 32757, 63545, 11564, 32888, 52548, 1116, 30] avg=32768.0000 max=67696 max_rank=0 min=30 min_rank=7 imbalance_ratio=2.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=30942 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=29308 max_load_reduction_pct=0.469634 moved_to_replicas=68427 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] avg=32768.0000 max=33097 max_rank=2 min=31252 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=59754 max_load_reduction_pct=0.643547 moved_to_replicas=103518 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[67696, 32757, 63545, 11564, 32888, 52548, 1116, 30] avg=32768.0000 max=67696 max_rank=0 min=30 min_rank=7 imbalance_ratio=2.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[67249, 97255, 126, 64732, 36, 23, 10, 32713] avg=32768.0000 max=97255 max_rank=1 min=10 min_rank=6 imbalance_ratio=2.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 32841, 31252, 33096, 32570, 33096] avg=32768.0000 max=33097 max_rank=2 min=31252 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=59754 max_load_reduction_pct=0.643547 moved_to_replicas=103518 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31366 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34600 max_load_reduction_pct=0.511108 moved_to_replicas=84501 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[67696, 32757, 63545, 11564, 32888, 52548, 1116, 30] avg=32768.0000 max=67696 max_rank=0 min=30 min_rank=7 imbalance_ratio=2.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31366 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34600 max_load_reduction_pct=0.511108 moved_to_replicas=84501 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31366 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34600 max_load_reduction_pct=0.511108 moved_to_replicas=84501 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] avg=32768.0000 max=33096 max_rank=0 min=31685 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64159 max_load_reduction_pct=0.659699 moved_to_replicas=129948 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[67696, 32757, 63545, 11564, 32888, 52548, 1116, 30] avg=32768.0000 max=67696 max_rank=0 min=30 min_rank=7 imbalance_ratio=2.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[0, 51029, 62406, 27616, 32836, 32886, 1091, 54280] avg=32768.0000 max=62406 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.904480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31366 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34600 max_load_reduction_pct=0.511108 moved_to_replicas=84501 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[0, 51029, 62406, 27616, 32836, 32886, 1091, 54280] avg=32768.0000 max=62406 max_rank=2 min=0 min_rank=0 imbalance_ratio=1.904480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[67249, 97255, 126, 64732, 36, 23, 10, 32713] avg=32768.0000 max=97255 max_rank=1 min=10 min_rank=6 imbalance_ratio=2.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31366 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34600 max_load_reduction_pct=0.511108 moved_to_replicas=84501 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[67249, 97255, 126, 64732, 36, 23, 10, 32713] avg=32768.0000 max=97255 max_rank=1 min=10 min_rank=6 imbalance_ratio=2.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[67249, 97255, 126, 64732, 36, 23, 10, 32713] avg=32768.0000 max=97255 max_rank=1 min=10 min_rank=6 imbalance_ratio=2.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=30942 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=29308 max_load_reduction_pct=0.469634 moved_to_replicas=68427 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[5, 32276, 45795, 34202, 63, 64951, 32735, 52117] avg=32768.0000 max=64951 max_rank=5 min=5 min_rank=0 imbalance_ratio=1.982147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[30942, 33095, 33096, 33097, 32836, 32886, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=30942 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=29308 max_load_reduction_pct=0.469634 moved_to_replicas=68427 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] avg=32768.0000 max=33096 max_rank=0 min=31685 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64159 max_load_reduction_pct=0.659699 moved_to_replicas=129948 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[67249, 97255, 126, 64732, 36, 23, 10, 32713] avg=32768.0000 max=97255 max_rank=1 min=10 min_rank=6 imbalance_ratio=2.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] avg=32768.0000 max=33096 max_rank=0 min=31685 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64159 max_load_reduction_pct=0.659699 moved_to_replicas=129948 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] avg=32768.0000 max=33096 max_rank=0 min=31685 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64159 max_load_reduction_pct=0.659699 moved_to_replicas=129948 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31860 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31854 max_load_reduction_pct=0.490431 moved_to_replicas=64681 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[67249, 97255, 126, 64732, 36, 23, 10, 32713] avg=32768.0000 max=97255 max_rank=1 min=10 min_rank=6 imbalance_ratio=2.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[67696, 32757, 63545, 11564, 32888, 52548, 1116, 30] avg=32768.0000 max=67696 max_rank=0 min=30 min_rank=7 imbalance_ratio=2.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] avg=32768.0000 max=33096 max_rank=0 min=31685 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64159 max_load_reduction_pct=0.659699 moved_to_replicas=129948 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] avg=32768.0000 max=33096 max_rank=0 min=31685 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64159 max_load_reduction_pct=0.659699 moved_to_replicas=129948 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[67696, 32757, 63545, 11564, 32888, 52548, 1116, 30] avg=32768.0000 max=67696 max_rank=0 min=30 min_rank=7 imbalance_ratio=2.065918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[5, 32276, 45795, 34202, 63, 64951, 32735, 52117] avg=32768.0000 max=64951 max_rank=5 min=5 min_rank=0 imbalance_ratio=1.982147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[5, 32276, 45795, 34202, 63, 64951, 32735, 52117] avg=32768.0000 max=64951 max_rank=5 min=5 min_rank=0 imbalance_ratio=1.982147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31366 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34600 max_load_reduction_pct=0.511108 moved_to_replicas=84501 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[5, 32276, 45795, 34202, 63, 64951, 32735, 52117] avg=32768.0000 max=64951 max_rank=5 min=5 min_rank=0 imbalance_ratio=1.982147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[21265, 13990, 78108, 50843, 32693, 32768, 32406, 71] avg=32768.0000 max=78108 max_rank=2 min=71 min_rank=7 imbalance_ratio=2.383667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32757, 33096, 31366, 32888, 33095, 33096, 32750] avg=32768.0000 max=33096 max_rank=0 min=31366 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=34600 max_load_reduction_pct=0.511108 moved_to_replicas=84501 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31860 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31854 max_load_reduction_pct=0.490431 moved_to_replicas=64681 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[5, 32276, 45795, 34202, 63, 64951, 32735, 52117] avg=32768.0000 max=64951 max_rank=5 min=5 min_rank=0 imbalance_ratio=1.982147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31860 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31854 max_load_reduction_pct=0.490431 moved_to_replicas=64681 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[5, 32276, 45795, 34202, 63, 64951, 32735, 52117] avg=32768.0000 max=64951 max_rank=5 min=5 min_rank=0 imbalance_ratio=1.982147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31860 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31854 max_load_reduction_pct=0.490431 moved_to_replicas=64681 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45011 max_load_reduction_pct=0.576266 moved_to_replicas=62759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[67249, 97255, 126, 64732, 36, 23, 10, 32713] avg=32768.0000 max=97255 max_rank=1 min=10 min_rank=6 imbalance_ratio=2.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31860 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31854 max_load_reduction_pct=0.490431 moved_to_replicas=64681 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31860 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31854 max_load_reduction_pct=0.490431 moved_to_replicas=64681 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[21265, 13990, 78108, 50843, 32693, 32768, 32406, 71] avg=32768.0000 max=78108 max_rank=2 min=71 min_rank=7 imbalance_ratio=2.383667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[67249, 97255, 126, 64732, 36, 23, 10, 32713] avg=32768.0000 max=97255 max_rank=1 min=10 min_rank=6 imbalance_ratio=2.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[21265, 13990, 78108, 50843, 32693, 32768, 32406, 71] avg=32768.0000 max=78108 max_rank=2 min=71 min_rank=7 imbalance_ratio=2.383667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[30357, 32692, 4193, 31616, 65461, 32079, 65719, 27] avg=32768.0000 max=65719 max_rank=6 min=27 min_rank=7 imbalance_ratio=2.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] avg=32768.0000 max=33096 max_rank=0 min=31685 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64159 max_load_reduction_pct=0.659699 moved_to_replicas=129948 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[21265, 13990, 78108, 50843, 32693, 32768, 32406, 71] avg=32768.0000 max=78108 max_rank=2 min=71 min_rank=7 imbalance_ratio=2.383667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45011 max_load_reduction_pct=0.576266 moved_to_replicas=62759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32603, 31685, 32759, 32713] avg=32768.0000 max=33096 max_rank=0 min=31685 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64159 max_load_reduction_pct=0.659699 moved_to_replicas=129948 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[21265, 13990, 78108, 50843, 32693, 32768, 32406, 71] avg=32768.0000 max=78108 max_rank=2 min=71 min_rank=7 imbalance_ratio=2.383667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45011 max_load_reduction_pct=0.576266 moved_to_replicas=62759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] avg=32768.0000 max=33096 max_rank=0 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32623 max_load_reduction_pct=0.496401 moved_to_replicas=64988 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[21265, 13990, 78108, 50843, 32693, 32768, 32406, 71] avg=32768.0000 max=78108 max_rank=2 min=71 min_rank=7 imbalance_ratio=2.383667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45011 max_load_reduction_pct=0.576266 moved_to_replicas=62759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[5, 32276, 45795, 34202, 63, 64951, 32735, 52117] avg=32768.0000 max=64951 max_rank=5 min=5 min_rank=0 imbalance_ratio=1.982147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45011 max_load_reduction_pct=0.576266 moved_to_replicas=62759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45011 max_load_reduction_pct=0.576266 moved_to_replicas=62759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[30357, 32692, 4193, 31616, 65461, 32079, 65719, 27] avg=32768.0000 max=65719 max_rank=6 min=27 min_rank=7 imbalance_ratio=2.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[30357, 32692, 4193, 31616, 65461, 32079, 65719, 27] avg=32768.0000 max=65719 max_rank=6 min=27 min_rank=7 imbalance_ratio=2.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[5, 32276, 45795, 34202, 63, 64951, 32735, 52117] avg=32768.0000 max=64951 max_rank=5 min=5 min_rank=0 imbalance_ratio=1.982147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32768, 107, 38847, 32746, 33837, 90877, 194, 32768] avg=32768.0000 max=90877 max_rank=5 min=107 min_rank=1 imbalance_ratio=2.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[30357, 32692, 4193, 31616, 65461, 32079, 65719, 27] avg=32768.0000 max=65719 max_rank=6 min=27 min_rank=7 imbalance_ratio=2.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31860 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31854 max_load_reduction_pct=0.490431 moved_to_replicas=64681 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] avg=32768.0000 max=33096 max_rank=0 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32623 max_load_reduction_pct=0.496401 moved_to_replicas=64988 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] avg=32768.0000 max=33096 max_rank=0 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32623 max_load_reduction_pct=0.496401 moved_to_replicas=64988 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[30357, 32692, 4193, 31616, 65461, 32079, 65719, 27] avg=32768.0000 max=65719 max_rank=6 min=27 min_rank=7 imbalance_ratio=2.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[31860, 32276, 33096, 33096, 32888, 33096, 32735, 33097] avg=32768.0000 max=33097 max_rank=7 min=31860 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31854 max_load_reduction_pct=0.490431 moved_to_replicas=64681 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] avg=32768.0000 max=33096 max_rank=0 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32623 max_load_reduction_pct=0.496401 moved_to_replicas=64988 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] avg=32768.0000 max=33097 max_rank=5 min=31729 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57780 max_load_reduction_pct=0.635804 moved_to_replicas=64273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[30357, 32692, 4193, 31616, 65461, 32079, 65719, 27] avg=32768.0000 max=65719 max_rank=6 min=27 min_rank=7 imbalance_ratio=2.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[21265, 13990, 78108, 50843, 32693, 32768, 32406, 71] avg=32768.0000 max=78108 max_rank=2 min=71 min_rank=7 imbalance_ratio=2.383667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] avg=32768.0000 max=33096 max_rank=0 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32623 max_load_reduction_pct=0.496401 moved_to_replicas=64988 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32768, 107, 38847, 32746, 33837, 90877, 194, 32768] avg=32768.0000 max=90877 max_rank=5 min=107 min_rank=1 imbalance_ratio=2.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] avg=32768.0000 max=33096 max_rank=0 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32623 max_load_reduction_pct=0.496401 moved_to_replicas=64988 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32768, 107, 38847, 32746, 33837, 90877, 194, 32768] avg=32768.0000 max=90877 max_rank=5 min=107 min_rank=1 imbalance_ratio=2.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[21265, 13990, 78108, 50843, 32693, 32768, 32406, 71] avg=32768.0000 max=78108 max_rank=2 min=71 min_rank=7 imbalance_ratio=2.383667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32768, 107, 38847, 32746, 33837, 90877, 194, 32768] avg=32768.0000 max=90877 max_rank=5 min=107 min_rank=1 imbalance_ratio=2.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[97479, 32790, 215, 20863, 32731, 32740, 12569, 32757] avg=32768.0000 max=97479 max_rank=0 min=215 min_rank=2 imbalance_ratio=2.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45011 max_load_reduction_pct=0.576266 moved_to_replicas=62759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] avg=32768.0000 max=33097 max_rank=5 min=31729 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57780 max_load_reduction_pct=0.635804 moved_to_replicas=64273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] avg=32768.0000 max=33097 max_rank=5 min=31729 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57780 max_load_reduction_pct=0.635804 moved_to_replicas=64273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32768, 107, 38847, 32746, 33837, 90877, 194, 32768] avg=32768.0000 max=90877 max_rank=5 min=107 min_rank=1 imbalance_ratio=2.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32168, 33097, 33096, 32693, 32768, 32406, 32820] avg=32768.0000 max=33097 max_rank=2 min=32168 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=45011 max_load_reduction_pct=0.576266 moved_to_replicas=62759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32768, 107, 38847, 32746, 33837, 90877, 194, 32768] avg=32768.0000 max=90877 max_rank=5 min=107 min_rank=1 imbalance_ratio=2.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] avg=32768.0000 max=33097 max_rank=5 min=31729 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57780 max_load_reduction_pct=0.635804 moved_to_replicas=64273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=31970 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64383 max_load_reduction_pct=0.660481 moved_to_replicas=64383 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[30357, 32692, 4193, 31616, 65461, 32079, 65719, 27] avg=32768.0000 max=65719 max_rank=6 min=27 min_rank=7 imbalance_ratio=2.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] avg=32768.0000 max=33097 max_rank=5 min=31729 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57780 max_load_reduction_pct=0.635804 moved_to_replicas=64273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[97479, 32790, 215, 20863, 32731, 32740, 12569, 32757] avg=32768.0000 max=97479 max_rank=0 min=215 min_rank=2 imbalance_ratio=2.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] avg=32768.0000 max=33097 max_rank=5 min=31729 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57780 max_load_reduction_pct=0.635804 moved_to_replicas=64273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[97479, 32790, 215, 20863, 32731, 32740, 12569, 32757] avg=32768.0000 max=97479 max_rank=0 min=215 min_rank=2 imbalance_ratio=2.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[30357, 32692, 4193, 31616, 65461, 32079, 65719, 27] avg=32768.0000 max=65719 max_rank=6 min=27 min_rank=7 imbalance_ratio=2.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32781, 101554, 32654, 45, 60212, 34873, 5, 20] avg=32768.0000 max=101554 max_rank=1 min=5 min_rank=6 imbalance_ratio=3.099182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[97479, 32790, 215, 20863, 32731, 32740, 12569, 32757] avg=32768.0000 max=97479 max_rank=0 min=215 min_rank=2 imbalance_ratio=2.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] avg=32768.0000 max=33096 max_rank=0 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32623 max_load_reduction_pct=0.496401 moved_to_replicas=64988 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=31970 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64383 max_load_reduction_pct=0.660481 moved_to_replicas=64383 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=31970 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64383 max_load_reduction_pct=0.660481 moved_to_replicas=64383 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[97479, 32790, 215, 20863, 32731, 32740, 12569, 32757] avg=32768.0000 max=97479 max_rank=0 min=215 min_rank=2 imbalance_ratio=2.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[33096, 32692, 33096, 32339, 33096, 32079, 33096, 32650] avg=32768.0000 max=33096 max_rank=0 min=32079 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32623 max_load_reduction_pct=0.496401 moved_to_replicas=64988 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] avg=32768.0000 max=33097 max_rank=4 min=31898 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68457 max_load_reduction_pct=0.674095 moved_to_replicas=97351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=31970 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64383 max_load_reduction_pct=0.660481 moved_to_replicas=64383 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[97479, 32790, 215, 20863, 32731, 32740, 12569, 32757] avg=32768.0000 max=97479 max_rank=0 min=215 min_rank=2 imbalance_ratio=2.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=31970 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64383 max_load_reduction_pct=0.660481 moved_to_replicas=64383 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32768, 107, 38847, 32746, 33837, 90877, 194, 32768] avg=32768.0000 max=90877 max_rank=5 min=107 min_rank=1 imbalance_ratio=2.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32781, 101554, 32654, 45, 60212, 34873, 5, 20] avg=32768.0000 max=101554 max_rank=1 min=5 min_rank=6 imbalance_ratio=3.099182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32781, 101554, 32654, 45, 60212, 34873, 5, 20] avg=32768.0000 max=101554 max_rank=1 min=5 min_rank=6 imbalance_ratio=3.099182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=31970 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64383 max_load_reduction_pct=0.660481 moved_to_replicas=64383 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[408, 0, 97962, 41544, 31060, 65529, 23802, 1839] avg=32768.0000 max=97962 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.989563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32768, 107, 38847, 32746, 33837, 90877, 194, 32768] avg=32768.0000 max=90877 max_rank=5 min=107 min_rank=1 imbalance_ratio=2.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32781, 101554, 32654, 45, 60212, 34873, 5, 20] avg=32768.0000 max=101554 max_rank=1 min=5 min_rank=6 imbalance_ratio=3.099182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] avg=32768.0000 max=33097 max_rank=5 min=31729 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57780 max_load_reduction_pct=0.635804 moved_to_replicas=64273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] avg=32768.0000 max=33097 max_rank=4 min=31898 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68457 max_load_reduction_pct=0.674095 moved_to_replicas=97351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] avg=32768.0000 max=33097 max_rank=4 min=31898 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68457 max_load_reduction_pct=0.674095 moved_to_replicas=97351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32781, 101554, 32654, 45, 60212, 34873, 5, 20] avg=32768.0000 max=101554 max_rank=1 min=5 min_rank=6 imbalance_ratio=3.099182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31390 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64866 max_load_reduction_pct=0.662155 moved_to_replicas=105747 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32768, 32845, 33095, 32746, 33096, 33097, 31729, 32768] avg=32768.0000 max=33097 max_rank=5 min=31729 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57780 max_load_reduction_pct=0.635804 moved_to_replicas=64273 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] avg=32768.0000 max=33097 max_rank=4 min=31898 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68457 max_load_reduction_pct=0.674095 moved_to_replicas=97351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32781, 101554, 32654, 45, 60212, 34873, 5, 20] avg=32768.0000 max=101554 max_rank=1 min=5 min_rank=6 imbalance_ratio=3.099182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[97479, 32790, 215, 20863, 32731, 32740, 12569, 32757] avg=32768.0000 max=97479 max_rank=0 min=215 min_rank=2 imbalance_ratio=2.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] avg=32768.0000 max=33097 max_rank=4 min=31898 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68457 max_load_reduction_pct=0.674095 moved_to_replicas=97351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[408, 0, 97962, 41544, 31060, 65529, 23802, 1839] avg=32768.0000 max=97962 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.989563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[408, 0, 97962, 41544, 31060, 65529, 23802, 1839] avg=32768.0000 max=97962 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.989563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] avg=32768.0000 max=33097 max_rank=4 min=31898 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68457 max_load_reduction_pct=0.674095 moved_to_replicas=97351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[32769, 27995, 305, 62365, 50, 42296, 63613, 32751] avg=32768.0000 max=63613 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.941315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[97479, 32790, 215, 20863, 32731, 32740, 12569, 32757] avg=32768.0000 max=97479 max_rank=0 min=215 min_rank=2 imbalance_ratio=2.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[408, 0, 97962, 41544, 31060, 65529, 23802, 1839] avg=32768.0000 max=97962 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.989563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=31970 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64383 max_load_reduction_pct=0.660481 moved_to_replicas=64383 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31390 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64866 max_load_reduction_pct=0.662155 moved_to_replicas=105747 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31390 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64866 max_load_reduction_pct=0.662155 moved_to_replicas=105747 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[408, 0, 97962, 41544, 31060, 65529, 23802, 1839] avg=32768.0000 max=97962 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.989563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] avg=32768.0000 max=33096 max_rank=1 min=31144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30517 max_load_reduction_pct=0.479729 moved_to_replicas=68986 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[33096, 32790, 32964, 31970, 32731, 32740, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=31970 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64383 max_load_reduction_pct=0.660481 moved_to_replicas=64383 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31390 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64866 max_load_reduction_pct=0.662155 moved_to_replicas=105747 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[408, 0, 97962, 41544, 31060, 65529, 23802, 1839] avg=32768.0000 max=97962 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.989563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31390 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64866 max_load_reduction_pct=0.662155 moved_to_replicas=105747 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[32769, 27995, 305, 62365, 50, 42296, 63613, 32751] avg=32768.0000 max=63613 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.941315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[32769, 27995, 305, 62365, 50, 42296, 63613, 32751] avg=32768.0000 max=63613 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.941315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32781, 101554, 32654, 45, 60212, 34873, 5, 20] avg=32768.0000 max=101554 max_rank=1 min=5 min_rank=6 imbalance_ratio=3.099182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31390 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64866 max_load_reduction_pct=0.662155 moved_to_replicas=105747 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[24660, 41194, 32784, 65473, 36, 65228, 32750, 19] avg=32768.0000 max=65473 max_rank=3 min=19 min_rank=7 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32781, 101554, 32654, 45, 60212, 34873, 5, 20] avg=32768.0000 max=101554 max_rank=1 min=5 min_rank=6 imbalance_ratio=3.099182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[32769, 27995, 305, 62365, 50, 42296, 63613, 32751] avg=32768.0000 max=63613 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.941315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] avg=32768.0000 max=33096 max_rank=1 min=31144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30517 max_load_reduction_pct=0.479729 moved_to_replicas=68986 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] avg=32768.0000 max=33096 max_rank=1 min=31144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30517 max_load_reduction_pct=0.479729 moved_to_replicas=68986 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] avg=32768.0000 max=33097 max_rank=4 min=31898 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68457 max_load_reduction_pct=0.674095 moved_to_replicas=97351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[32769, 27995, 305, 62365, 50, 42296, 63613, 32751] avg=32768.0000 max=63613 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.941315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] avg=32768.0000 max=33096 max_rank=1 min=32168 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=72607 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32781, 33096, 32654, 31898, 33097, 33096, 32754, 32768] avg=32768.0000 max=33097 max_rank=4 min=31898 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68457 max_load_reduction_pct=0.674095 moved_to_replicas=97351 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] avg=32768.0000 max=33096 max_rank=1 min=31144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30517 max_load_reduction_pct=0.479729 moved_to_replicas=68986 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[32769, 27995, 305, 62365, 50, 42296, 63613, 32751] avg=32768.0000 max=63613 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.941315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] avg=32768.0000 max=33096 max_rank=1 min=31144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30517 max_load_reduction_pct=0.479729 moved_to_replicas=68986 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[24660, 41194, 32784, 65473, 36, 65228, 32750, 19] avg=32768.0000 max=65473 max_rank=3 min=19 min_rank=7 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[24660, 41194, 32784, 65473, 36, 65228, 32750, 19] avg=32768.0000 max=65473 max_rank=3 min=19 min_rank=7 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[408, 0, 97962, 41544, 31060, 65529, 23802, 1839] avg=32768.0000 max=97962 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.989563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] avg=32768.0000 max=33096 max_rank=1 min=31144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30517 max_load_reduction_pct=0.479729 moved_to_replicas=68986 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[4, 24179, 63103, 9370, 32768, 32734, 32801, 67185] avg=32768.0000 max=67185 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.050323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[408, 0, 97962, 41544, 31060, 65529, 23802, 1839] avg=32768.0000 max=97962 max_rank=2 min=0 min_rank=1 imbalance_ratio=2.989563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[24660, 41194, 32784, 65473, 36, 65228, 32750, 19] avg=32768.0000 max=65473 max_rank=3 min=19 min_rank=7 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] avg=32768.0000 max=33096 max_rank=1 min=32168 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=72607 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] avg=32768.0000 max=33096 max_rank=1 min=32168 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=72607 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31390 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64866 max_load_reduction_pct=0.662155 moved_to_replicas=105747 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] avg=32768.0000 max=33097 max_rank=3 min=31799 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=34088 max_load_reduction_pct=0.507375 moved_to_replicas=64096 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[24660, 41194, 32784, 65473, 36, 65228, 32750, 19] avg=32768.0000 max=65473 max_rank=3 min=19 min_rank=7 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[32528, 32746, 33096, 33096, 31390, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31390 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64866 max_load_reduction_pct=0.662155 moved_to_replicas=105747 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[24660, 41194, 32784, 65473, 36, 65228, 32750, 19] avg=32768.0000 max=65473 max_rank=3 min=19 min_rank=7 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] avg=32768.0000 max=33096 max_rank=1 min=32168 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=72607 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[4, 24179, 63103, 9370, 32768, 32734, 32801, 67185] avg=32768.0000 max=67185 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.050323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] avg=32768.0000 max=33096 max_rank=1 min=32168 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=72607 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[4, 24179, 63103, 9370, 32768, 32734, 32801, 67185] avg=32768.0000 max=67185 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.050323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[32769, 27995, 305, 62365, 50, 42296, 63613, 32751] avg=32768.0000 max=63613 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.941315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[36, 32806, 55604, 42264, 38, 0, 32766, 98630] avg=32768.0000 max=98630 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] avg=32768.0000 max=33096 max_rank=1 min=32168 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=72607 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[32769, 27995, 305, 62365, 50, 42296, 63613, 32751] avg=32768.0000 max=63613 max_rank=6 min=50 min_rank=4 imbalance_ratio=1.941315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[4, 24179, 63103, 9370, 32768, 32734, 32801, 67185] avg=32768.0000 max=67185 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.050323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] avg=32768.0000 max=33097 max_rank=3 min=31799 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=34088 max_load_reduction_pct=0.507375 moved_to_replicas=64096 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] avg=32768.0000 max=33097 max_rank=3 min=31799 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=34088 max_load_reduction_pct=0.507375 moved_to_replicas=64096 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] avg=32768.0000 max=33096 max_rank=1 min=31144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30517 max_load_reduction_pct=0.479729 moved_to_replicas=68986 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65534 max_load_reduction_pct=0.664443 moved_to_replicas=97210 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[4, 24179, 63103, 9370, 32768, 32734, 32801, 67185] avg=32768.0000 max=67185 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.050323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[32769, 33096, 33096, 33096, 31144, 33096, 33096, 32751] avg=32768.0000 max=33096 max_rank=1 min=31144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30517 max_load_reduction_pct=0.479729 moved_to_replicas=68986 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] avg=32768.0000 max=33097 max_rank=3 min=31799 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=34088 max_load_reduction_pct=0.507375 moved_to_replicas=64096 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[4, 24179, 63103, 9370, 32768, 32734, 32801, 67185] avg=32768.0000 max=67185 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.050323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[36, 32806, 55604, 42264, 38, 0, 32766, 98630] avg=32768.0000 max=98630 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] avg=32768.0000 max=33097 max_rank=3 min=31799 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=34088 max_load_reduction_pct=0.507375 moved_to_replicas=64096 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[36, 32806, 55604, 42264, 38, 0, 32766, 98630] avg=32768.0000 max=98630 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[61, 98218, 358, 32330, 65841, 128, 30670, 34538] avg=32768.0000 max=98218 max_rank=1 min=61 min_rank=0 imbalance_ratio=2.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[24660, 41194, 32784, 65473, 36, 65228, 32750, 19] avg=32768.0000 max=65473 max_rank=3 min=19 min_rank=7 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] avg=32768.0000 max=33097 max_rank=3 min=31799 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=34088 max_load_reduction_pct=0.507375 moved_to_replicas=64096 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[24660, 41194, 32784, 65473, 36, 65228, 32750, 19] avg=32768.0000 max=65473 max_rank=3 min=19 min_rank=7 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[36, 32806, 55604, 42264, 38, 0, 32766, 98630] avg=32768.0000 max=98630 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65534 max_load_reduction_pct=0.664443 moved_to_replicas=97210 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65534 max_load_reduction_pct=0.664443 moved_to_replicas=97210 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65122 max_load_reduction_pct=0.663035 moved_to_replicas=99309 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] avg=32768.0000 max=33096 max_rank=1 min=32168 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=72607 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[36, 32806, 55604, 42264, 38, 0, 32766, 98630] avg=32768.0000 max=98630 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[36, 32806, 55604, 42264, 38, 0, 32766, 98630] avg=32768.0000 max=98630 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[32758, 33096, 32784, 33096, 32168, 33096, 32750, 32396] avg=32768.0000 max=33096 max_rank=1 min=32168 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=72607 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65534 max_load_reduction_pct=0.664443 moved_to_replicas=97210 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65534 max_load_reduction_pct=0.664443 moved_to_replicas=97210 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[61, 98218, 358, 32330, 65841, 128, 30670, 34538] avg=32768.0000 max=98218 max_rank=1 min=61 min_rank=0 imbalance_ratio=2.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[61, 98218, 358, 32330, 65841, 128, 30670, 34538] avg=32768.0000 max=98218 max_rank=1 min=61 min_rank=0 imbalance_ratio=2.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[18, 4410, 32750, 65512, 65965, 5254, 87357, 878] avg=32768.0000 max=87357 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.665924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[4, 24179, 63103, 9370, 32768, 32734, 32801, 67185] avg=32768.0000 max=67185 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.050323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65534 max_load_reduction_pct=0.664443 moved_to_replicas=97210 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[4, 24179, 63103, 9370, 32768, 32734, 32801, 67185] avg=32768.0000 max=67185 max_rank=7 min=4 min_rank=0 imbalance_ratio=2.050323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[61, 98218, 358, 32330, 65841, 128, 30670, 34538] avg=32768.0000 max=98218 max_rank=1 min=61 min_rank=0 imbalance_ratio=2.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65122 max_load_reduction_pct=0.663035 moved_to_replicas=99309 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65122 max_load_reduction_pct=0.663035 moved_to_replicas=99309 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] avg=32768.0000 max=33096 max_rank=1 min=31152 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=54261 max_load_reduction_pct=0.621141 moved_to_replicas=119546 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] avg=32768.0000 max=33097 max_rank=3 min=31799 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=34088 max_load_reduction_pct=0.507375 moved_to_replicas=64096 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[61, 98218, 358, 32330, 65841, 128, 30670, 34538] avg=32768.0000 max=98218 max_rank=1 min=61 min_rank=0 imbalance_ratio=2.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[32753, 31799, 33096, 33097, 32768, 32734, 32801, 33096] avg=32768.0000 max=33097 max_rank=3 min=31799 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=34088 max_load_reduction_pct=0.507375 moved_to_replicas=64096 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[61, 98218, 358, 32330, 65841, 128, 30670, 34538] avg=32768.0000 max=98218 max_rank=1 min=61 min_rank=0 imbalance_ratio=2.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65122 max_load_reduction_pct=0.663035 moved_to_replicas=99309 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[18, 4410, 32750, 65512, 65965, 5254, 87357, 878] avg=32768.0000 max=87357 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.665924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65122 max_load_reduction_pct=0.663035 moved_to_replicas=99309 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[18, 4410, 32750, 65512, 65965, 5254, 87357, 878] avg=32768.0000 max=87357 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.665924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[32752, 61011, 32724, 7990, 62104, 10, 32749, 32804] avg=32768.0000 max=62104 max_rank=4 min=10 min_rank=5 imbalance_ratio=1.895264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[36, 32806, 55604, 42264, 38, 0, 32766, 98630] avg=32768.0000 max=98630 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65122 max_load_reduction_pct=0.663035 moved_to_replicas=99309 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[36, 32806, 55604, 42264, 38, 0, 32766, 98630] avg=32768.0000 max=98630 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] avg=32768.0000 max=33096 max_rank=1 min=31152 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=54261 max_load_reduction_pct=0.621141 moved_to_replicas=119546 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[18, 4410, 32750, 65512, 65965, 5254, 87357, 878] avg=32768.0000 max=87357 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.665924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] avg=32768.0000 max=33096 max_rank=1 min=31152 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=54261 max_load_reduction_pct=0.621141 moved_to_replicas=119546 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] avg=32768.0000 max=33096 max_rank=1 min=31827 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29008 max_load_reduction_pct=0.467087 moved_to_replicas=56923 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65534 max_load_reduction_pct=0.664443 moved_to_replicas=97210 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[18, 4410, 32750, 65512, 65965, 5254, 87357, 878] avg=32768.0000 max=87357 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.665924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[32783, 32806, 33096, 33095, 31753, 32749, 32766, 33096] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65534 max_load_reduction_pct=0.664443 moved_to_replicas=97210 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[18, 4410, 32750, 65512, 65965, 5254, 87357, 878] avg=32768.0000 max=87357 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.665924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] avg=32768.0000 max=33096 max_rank=1 min=31152 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=54261 max_load_reduction_pct=0.621141 moved_to_replicas=119546 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[32752, 61011, 32724, 7990, 62104, 10, 32749, 32804] avg=32768.0000 max=62104 max_rank=4 min=10 min_rank=5 imbalance_ratio=1.895264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] avg=32768.0000 max=33096 max_rank=1 min=31152 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=54261 max_load_reduction_pct=0.621141 moved_to_replicas=119546 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[32752, 61011, 32724, 7990, 62104, 10, 32749, 32804] avg=32768.0000 max=62104 max_rank=4 min=10 min_rank=5 imbalance_ratio=1.895264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[18, 14524, 32781, 65344, 24, 65413, 33602, 50438] avg=32768.0000 max=65413 max_rank=5 min=18 min_rank=0 imbalance_ratio=1.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] avg=32768.0000 max=33096 max_rank=1 min=31152 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=54261 max_load_reduction_pct=0.621141 moved_to_replicas=119546 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[61, 98218, 358, 32330, 65841, 128, 30670, 34538] avg=32768.0000 max=98218 max_rank=1 min=61 min_rank=0 imbalance_ratio=2.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] avg=32768.0000 max=33096 max_rank=1 min=31827 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29008 max_load_reduction_pct=0.467087 moved_to_replicas=56923 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[61, 98218, 358, 32330, 65841, 128, 30670, 34538] avg=32768.0000 max=98218 max_rank=1 min=61 min_rank=0 imbalance_ratio=2.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[32752, 61011, 32724, 7990, 62104, 10, 32749, 32804] avg=32768.0000 max=62104 max_rank=4 min=10 min_rank=5 imbalance_ratio=1.895264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] avg=32768.0000 max=33096 max_rank=1 min=31827 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29008 max_load_reduction_pct=0.467087 moved_to_replicas=56923 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=32272 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32317 max_load_reduction_pct=0.494046 moved_to_replicas=82413 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65122 max_load_reduction_pct=0.663035 moved_to_replicas=99309 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[32752, 61011, 32724, 7990, 62104, 10, 32749, 32804] avg=32768.0000 max=62104 max_rank=4 min=10 min_rank=5 imbalance_ratio=1.895264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[32752, 61011, 32724, 7990, 62104, 10, 32749, 32804] avg=32768.0000 max=62104 max_rank=4 min=10 min_rank=5 imbalance_ratio=1.895264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[32810, 33096, 33074, 32330, 33096, 32501, 32141, 33096] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65122 max_load_reduction_pct=0.663035 moved_to_replicas=99309 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] avg=32768.0000 max=33096 max_rank=1 min=31827 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29008 max_load_reduction_pct=0.467087 moved_to_replicas=56923 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[18, 14524, 32781, 65344, 24, 65413, 33602, 50438] avg=32768.0000 max=65413 max_rank=5 min=18 min_rank=0 imbalance_ratio=1.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] avg=32768.0000 max=33096 max_rank=1 min=31827 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29008 max_load_reduction_pct=0.467087 moved_to_replicas=56923 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[18, 14524, 32781, 65344, 24, 65413, 33602, 50438] avg=32768.0000 max=65413 max_rank=5 min=18 min_rank=0 imbalance_ratio=1.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=25 tokens_per_ep_rank=[21421, 32733, 51648, 65476, 66747, 18, 24050, 51] avg=32768.0000 max=66747 max_rank=4 min=18 min_rank=5 imbalance_ratio=2.036957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[18, 4410, 32750, 65512, 65965, 5254, 87357, 878] avg=32768.0000 max=87357 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.665924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] avg=32768.0000 max=33096 max_rank=1 min=31827 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29008 max_load_reduction_pct=0.467087 moved_to_replicas=56923 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=32272 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32317 max_load_reduction_pct=0.494046 moved_to_replicas=82413 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[18, 4410, 32750, 65512, 65965, 5254, 87357, 878] avg=32768.0000 max=87357 max_rank=6 min=18 min_rank=0 imbalance_ratio=2.665924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=32272 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32317 max_load_reduction_pct=0.494046 moved_to_replicas=82413 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[18, 14524, 32781, 65344, 24, 65413, 33602, 50438] avg=32768.0000 max=65413 max_rank=5 min=18 min_rank=0 imbalance_ratio=1.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=25 tokens_per_ep_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] avg=32768.0000 max=33097 max_rank=2 min=31164 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33650 max_load_reduction_pct=0.504143 moved_to_replicas=84583 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[18, 14524, 32781, 65344, 24, 65413, 33602, 50438] avg=32768.0000 max=65413 max_rank=5 min=18 min_rank=0 imbalance_ratio=1.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] avg=32768.0000 max=33096 max_rank=1 min=31152 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=54261 max_load_reduction_pct=0.621141 moved_to_replicas=119546 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[18, 14524, 32781, 65344, 24, 65413, 33602, 50438] avg=32768.0000 max=65413 max_rank=5 min=18 min_rank=0 imbalance_ratio=1.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[32763, 33096, 32750, 33096, 33096, 33096, 33095, 31152] avg=32768.0000 max=33096 max_rank=1 min=31152 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=54261 max_load_reduction_pct=0.621141 moved_to_replicas=119546 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=32272 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32317 max_load_reduction_pct=0.494046 moved_to_replicas=82413 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=25 tokens_per_ep_rank=[21421, 32733, 51648, 65476, 66747, 18, 24050, 51] avg=32768.0000 max=66747 max_rank=4 min=18 min_rank=5 imbalance_ratio=2.036957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=32272 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32317 max_load_reduction_pct=0.494046 moved_to_replicas=82413 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=25 tokens_per_ep_rank=[21421, 32733, 51648, 65476, 66747, 18, 24050, 51] avg=32768.0000 max=66747 max_rank=4 min=18 min_rank=5 imbalance_ratio=2.036957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=26 tokens_per_ep_rank=[40802, 44146, 32705, 7, 6, 88705, 55684, 89] avg=32768.0000 max=88705 max_rank=5 min=6 min_rank=4 imbalance_ratio=2.707062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=32272 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32317 max_load_reduction_pct=0.494046 moved_to_replicas=82413 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[32752, 61011, 32724, 7990, 62104, 10, 32749, 32804] avg=32768.0000 max=62104 max_rank=4 min=10 min_rank=5 imbalance_ratio=1.895264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=25 tokens_per_ep_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] avg=32768.0000 max=33097 max_rank=2 min=31164 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33650 max_load_reduction_pct=0.504143 moved_to_replicas=84583 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[32752, 61011, 32724, 7990, 62104, 10, 32749, 32804] avg=32768.0000 max=62104 max_rank=4 min=10 min_rank=5 imbalance_ratio=1.895264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=25 tokens_per_ep_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] avg=32768.0000 max=33097 max_rank=2 min=31164 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33650 max_load_reduction_pct=0.504143 moved_to_replicas=84583 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=25 tokens_per_ep_rank=[21421, 32733, 51648, 65476, 66747, 18, 24050, 51] avg=32768.0000 max=66747 max_rank=4 min=18 min_rank=5 imbalance_ratio=2.036957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=26 tokens_per_ep_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31203 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=55608 max_load_reduction_pct=0.626887 moved_to_replicas=96953 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=25 tokens_per_ep_rank=[21421, 32733, 51648, 65476, 66747, 18, 24050, 51] avg=32768.0000 max=66747 max_rank=4 min=18 min_rank=5 imbalance_ratio=2.036957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] avg=32768.0000 max=33096 max_rank=1 min=31827 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29008 max_load_reduction_pct=0.467087 moved_to_replicas=56923 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=25 tokens_per_ep_rank=[21421, 32733, 51648, 65476, 66747, 18, 24050, 51] avg=32768.0000 max=66747 max_rank=4 min=18 min_rank=5 imbalance_ratio=2.036957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[32752, 33096, 32724, 33096, 33096, 31827, 32749, 32804] avg=32768.0000 max=33096 max_rank=1 min=31827 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=29008 max_load_reduction_pct=0.467087 moved_to_replicas=56923 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=25 tokens_per_ep_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] avg=32768.0000 max=33097 max_rank=2 min=31164 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33650 max_load_reduction_pct=0.504143 moved_to_replicas=84583 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=26 tokens_per_ep_rank=[40802, 44146, 32705, 7, 6, 88705, 55684, 89] avg=32768.0000 max=88705 max_rank=5 min=6 min_rank=4 imbalance_ratio=2.707062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=25 tokens_per_ep_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] avg=32768.0000 max=33097 max_rank=2 min=31164 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33650 max_load_reduction_pct=0.504143 moved_to_replicas=84583 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=26 tokens_per_ep_rank=[40802, 44146, 32705, 7, 6, 88705, 55684, 89] avg=32768.0000 max=88705 max_rank=5 min=6 min_rank=4 imbalance_ratio=2.707062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=27 tokens_per_ep_rank=[42, 36275, 10, 5798, 32741, 32770, 89033, 65475] avg=32768.0000 max=89033 max_rank=6 min=10 min_rank=2 imbalance_ratio=2.717072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=25 tokens_per_ep_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] avg=32768.0000 max=33097 max_rank=2 min=31164 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33650 max_load_reduction_pct=0.504143 moved_to_replicas=84583 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=26 tokens_per_ep_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31203 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=55608 max_load_reduction_pct=0.626887 moved_to_replicas=96953 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[18, 14524, 32781, 65344, 24, 65413, 33602, 50438] avg=32768.0000 max=65413 max_rank=5 min=18 min_rank=0 imbalance_ratio=1.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=26 tokens_per_ep_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31203 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=55608 max_load_reduction_pct=0.626887 moved_to_replicas=96953 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[18, 14524, 32781, 65344, 24, 65413, 33602, 50438] avg=32768.0000 max=65413 max_rank=5 min=18 min_rank=0 imbalance_ratio=1.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=26 tokens_per_ep_rank=[40802, 44146, 32705, 7, 6, 88705, 55684, 89] avg=32768.0000 max=88705 max_rank=5 min=6 min_rank=4 imbalance_ratio=2.707062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=27 tokens_per_ep_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31492 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55937 max_load_reduction_pct=0.628273 moved_to_replicas=91495 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=26 tokens_per_ep_rank=[40802, 44146, 32705, 7, 6, 88705, 55684, 89] avg=32768.0000 max=88705 max_rank=5 min=6 min_rank=4 imbalance_ratio=2.707062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=32272 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32317 max_load_reduction_pct=0.494046 moved_to_replicas=82413 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=26 tokens_per_ep_rank=[40802, 44146, 32705, 7, 6, 88705, 55684, 89] avg=32768.0000 max=88705 max_rank=5 min=6 min_rank=4 imbalance_ratio=2.707062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[32335, 32373, 32781, 33096, 32272, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=32272 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32317 max_load_reduction_pct=0.494046 moved_to_replicas=82413 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=27 tokens_per_ep_rank=[42, 36275, 10, 5798, 32741, 32770, 89033, 65475] avg=32768.0000 max=89033 max_rank=6 min=10 min_rank=2 imbalance_ratio=2.717072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=26 tokens_per_ep_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31203 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=55608 max_load_reduction_pct=0.626887 moved_to_replicas=96953 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=27 tokens_per_ep_rank=[42, 36275, 10, 5798, 32741, 32770, 89033, 65475] avg=32768.0000 max=89033 max_rank=6 min=10 min_rank=2 imbalance_ratio=2.717072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=26 tokens_per_ep_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31203 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=55608 max_load_reduction_pct=0.626887 moved_to_replicas=96953 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=26 tokens_per_ep_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31203 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=55608 max_load_reduction_pct=0.626887 moved_to_replicas=96953 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=28 tokens_per_ep_rank=[35074, 32754, 63547, 274, 32757, 64935, 18, 32785] avg=32768.0000 max=64935 max_rank=5 min=18 min_rank=6 imbalance_ratio=1.981659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=27 tokens_per_ep_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31492 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55937 max_load_reduction_pct=0.628273 moved_to_replicas=91495 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=25 tokens_per_ep_rank=[21421, 32733, 51648, 65476, 66747, 18, 24050, 51] avg=32768.0000 max=66747 max_rank=4 min=18 min_rank=5 imbalance_ratio=2.036957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=27 tokens_per_ep_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31492 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55937 max_load_reduction_pct=0.628273 moved_to_replicas=91495 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=25 tokens_per_ep_rank=[21421, 32733, 51648, 65476, 66747, 18, 24050, 51] avg=32768.0000 max=66747 max_rank=4 min=18 min_rank=5 imbalance_ratio=2.036957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=27 tokens_per_ep_rank=[42, 36275, 10, 5798, 32741, 32770, 89033, 65475] avg=32768.0000 max=89033 max_rank=6 min=10 min_rank=2 imbalance_ratio=2.717072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=28 tokens_per_ep_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31839 max_load_reduction_pct=0.490321 moved_to_replicas=64268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=27 tokens_per_ep_rank=[42, 36275, 10, 5798, 32741, 32770, 89033, 65475] avg=32768.0000 max=89033 max_rank=6 min=10 min_rank=2 imbalance_ratio=2.717072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=27 tokens_per_ep_rank=[42, 36275, 10, 5798, 32741, 32770, 89033, 65475] avg=32768.0000 max=89033 max_rank=6 min=10 min_rank=2 imbalance_ratio=2.717072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=25 tokens_per_ep_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] avg=32768.0000 max=33097 max_rank=2 min=31164 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33650 max_load_reduction_pct=0.504143 moved_to_replicas=84583 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=25 tokens_per_ep_rank=[33095, 32733, 33097, 33096, 33096, 32767, 31164, 33096] avg=32768.0000 max=33097 max_rank=2 min=31164 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33650 max_load_reduction_pct=0.504143 moved_to_replicas=84583 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=28 tokens_per_ep_rank=[35074, 32754, 63547, 274, 32757, 64935, 18, 32785] avg=32768.0000 max=64935 max_rank=5 min=18 min_rank=6 imbalance_ratio=1.981659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=27 tokens_per_ep_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31492 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55937 max_load_reduction_pct=0.628273 moved_to_replicas=91495 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=27 tokens_per_ep_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31492 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55937 max_load_reduction_pct=0.628273 moved_to_replicas=91495 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=28 tokens_per_ep_rank=[35074, 32754, 63547, 274, 32757, 64935, 18, 32785] avg=32768.0000 max=64935 max_rank=5 min=18 min_rank=6 imbalance_ratio=1.981659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=27 tokens_per_ep_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31492 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55937 max_load_reduction_pct=0.628273 moved_to_replicas=91495 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=29 tokens_per_ep_rank=[2640, 92051, 41, 15, 46546, 22737, 32783, 65331] avg=32768.0000 max=92051 max_rank=1 min=15 min_rank=3 imbalance_ratio=2.809174 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=28 tokens_per_ep_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31839 max_load_reduction_pct=0.490321 moved_to_replicas=64268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=26 tokens_per_ep_rank=[40802, 44146, 32705, 7, 6, 88705, 55684, 89] avg=32768.0000 max=88705 max_rank=5 min=6 min_rank=4 imbalance_ratio=2.707062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=28 tokens_per_ep_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31839 max_load_reduction_pct=0.490321 moved_to_replicas=64268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=26 tokens_per_ep_rank=[40802, 44146, 32705, 7, 6, 88705, 55684, 89] avg=32768.0000 max=88705 max_rank=5 min=6 min_rank=4 imbalance_ratio=2.707062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=29 tokens_per_ep_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] avg=32768.0000 max=33097 max_rank=5 min=31117 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=58954 max_load_reduction_pct=0.640449 moved_to_replicas=104640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=28 tokens_per_ep_rank=[35074, 32754, 63547, 274, 32757, 64935, 18, 32785] avg=32768.0000 max=64935 max_rank=5 min=18 min_rank=6 imbalance_ratio=1.981659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=28 tokens_per_ep_rank=[35074, 32754, 63547, 274, 32757, 64935, 18, 32785] avg=32768.0000 max=64935 max_rank=5 min=18 min_rank=6 imbalance_ratio=1.981659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=28 tokens_per_ep_rank=[35074, 32754, 63547, 274, 32757, 64935, 18, 32785] avg=32768.0000 max=64935 max_rank=5 min=18 min_rank=6 imbalance_ratio=1.981659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=26 tokens_per_ep_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31203 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=55608 max_load_reduction_pct=0.626887 moved_to_replicas=96953 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=26 tokens_per_ep_rank=[33096, 33095, 32705, 31203, 32755, 33097, 33096, 33097] avg=32768.0000 max=33097 max_rank=5 min=31203 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=55608 max_load_reduction_pct=0.626887 moved_to_replicas=96953 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=29 tokens_per_ep_rank=[2640, 92051, 41, 15, 46546, 22737, 32783, 65331] avg=32768.0000 max=92051 max_rank=1 min=15 min_rank=3 imbalance_ratio=2.809174 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=28 tokens_per_ep_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31839 max_load_reduction_pct=0.490321 moved_to_replicas=64268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=29 tokens_per_ep_rank=[2640, 92051, 41, 15, 46546, 22737, 32783, 65331] avg=32768.0000 max=92051 max_rank=1 min=15 min_rank=3 imbalance_ratio=2.809174 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=28 tokens_per_ep_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31839 max_load_reduction_pct=0.490321 moved_to_replicas=64268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=28 tokens_per_ep_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31839 max_load_reduction_pct=0.490321 moved_to_replicas=64268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=30 tokens_per_ep_rank=[32768, 67302, 18, 32796, 32777, 33983, 33025, 29475] avg=32768.0000 max=67302 max_rank=1 min=18 min_rank=2 imbalance_ratio=2.053894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=29 tokens_per_ep_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] avg=32768.0000 max=33097 max_rank=5 min=31117 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=58954 max_load_reduction_pct=0.640449 moved_to_replicas=104640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=27 tokens_per_ep_rank=[42, 36275, 10, 5798, 32741, 32770, 89033, 65475] avg=32768.0000 max=89033 max_rank=6 min=10 min_rank=2 imbalance_ratio=2.717072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=29 tokens_per_ep_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] avg=32768.0000 max=33097 max_rank=5 min=31117 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=58954 max_load_reduction_pct=0.640449 moved_to_replicas=104640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=30 tokens_per_ep_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34206 max_load_reduction_pct=0.508246 moved_to_replicas=35093 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=27 tokens_per_ep_rank=[42, 36275, 10, 5798, 32741, 32770, 89033, 65475] avg=32768.0000 max=89033 max_rank=6 min=10 min_rank=2 imbalance_ratio=2.717072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=29 tokens_per_ep_rank=[2640, 92051, 41, 15, 46546, 22737, 32783, 65331] avg=32768.0000 max=92051 max_rank=1 min=15 min_rank=3 imbalance_ratio=2.809174 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=29 tokens_per_ep_rank=[2640, 92051, 41, 15, 46546, 22737, 32783, 65331] avg=32768.0000 max=92051 max_rank=1 min=15 min_rank=3 imbalance_ratio=2.809174 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=29 tokens_per_ep_rank=[2640, 92051, 41, 15, 46546, 22737, 32783, 65331] avg=32768.0000 max=92051 max_rank=1 min=15 min_rank=3 imbalance_ratio=2.809174 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=27 tokens_per_ep_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31492 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55937 max_load_reduction_pct=0.628273 moved_to_replicas=91495 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=30 tokens_per_ep_rank=[32768, 67302, 18, 32796, 32777, 33983, 33025, 29475] avg=32768.0000 max=67302 max_rank=1 min=18 min_rank=2 imbalance_ratio=2.053894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=27 tokens_per_ep_rank=[31492, 33096, 32757, 33096, 32741, 32770, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31492 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55937 max_load_reduction_pct=0.628273 moved_to_replicas=91495 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=29 tokens_per_ep_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] avg=32768.0000 max=33097 max_rank=5 min=31117 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=58954 max_load_reduction_pct=0.640449 moved_to_replicas=104640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=29 tokens_per_ep_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] avg=32768.0000 max=33097 max_rank=5 min=31117 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=58954 max_load_reduction_pct=0.640449 moved_to_replicas=104640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=30 tokens_per_ep_rank=[32768, 67302, 18, 32796, 32777, 33983, 33025, 29475] avg=32768.0000 max=67302 max_rank=1 min=18 min_rank=2 imbalance_ratio=2.053894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=29 tokens_per_ep_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] avg=32768.0000 max=33097 max_rank=5 min=31117 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=58954 max_load_reduction_pct=0.640449 moved_to_replicas=104640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=31 tokens_per_ep_rank=[54, 98265, 112, 18, 21458, 32748, 43773, 65716] avg=32768.0000 max=98265 max_rank=1 min=18 min_rank=3 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=30 tokens_per_ep_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34206 max_load_reduction_pct=0.508246 moved_to_replicas=35093 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=30 tokens_per_ep_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34206 max_load_reduction_pct=0.508246 moved_to_replicas=35093 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=28 tokens_per_ep_rank=[35074, 32754, 63547, 274, 32757, 64935, 18, 32785] avg=32768.0000 max=64935 max_rank=5 min=18 min_rank=6 imbalance_ratio=1.981659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=31 tokens_per_ep_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65169 max_load_reduction_pct=0.663196 moved_to_replicas=108466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=28 tokens_per_ep_rank=[35074, 32754, 63547, 274, 32757, 64935, 18, 32785] avg=32768.0000 max=64935 max_rank=5 min=18 min_rank=6 imbalance_ratio=1.981659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=30 tokens_per_ep_rank=[32768, 67302, 18, 32796, 32777, 33983, 33025, 29475] avg=32768.0000 max=67302 max_rank=1 min=18 min_rank=2 imbalance_ratio=2.053894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=30 tokens_per_ep_rank=[32768, 67302, 18, 32796, 32777, 33983, 33025, 29475] avg=32768.0000 max=67302 max_rank=1 min=18 min_rank=2 imbalance_ratio=2.053894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=30 tokens_per_ep_rank=[32768, 67302, 18, 32796, 32777, 33983, 33025, 29475] avg=32768.0000 max=67302 max_rank=1 min=18 min_rank=2 imbalance_ratio=2.053894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=28 tokens_per_ep_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31839 max_load_reduction_pct=0.490321 moved_to_replicas=64268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=31 tokens_per_ep_rank=[54, 98265, 112, 18, 21458, 32748, 43773, 65716] avg=32768.0000 max=98265 max_rank=1 min=18 min_rank=3 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=28 tokens_per_ep_rank=[33096, 32754, 33096, 32703, 32757, 33096, 31857, 32785] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=31839 max_load_reduction_pct=0.490321 moved_to_replicas=64268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=30 tokens_per_ep_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34206 max_load_reduction_pct=0.508246 moved_to_replicas=35093 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=30 tokens_per_ep_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34206 max_load_reduction_pct=0.508246 moved_to_replicas=35093 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=31 tokens_per_ep_rank=[54, 98265, 112, 18, 21458, 32748, 43773, 65716] avg=32768.0000 max=98265 max_rank=1 min=18 min_rank=3 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=30 tokens_per_ep_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34206 max_load_reduction_pct=0.508246 moved_to_replicas=35093 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=32 tokens_per_ep_rank=[32, 18, 32895, 65518, 32611, 98308, 12, 32750] avg=32768.0000 max=98308 max_rank=5 min=12 min_rank=6 imbalance_ratio=3.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=31 tokens_per_ep_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65169 max_load_reduction_pct=0.663196 moved_to_replicas=108466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=31 tokens_per_ep_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65169 max_load_reduction_pct=0.663196 moved_to_replicas=108466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=29 tokens_per_ep_rank=[2640, 92051, 41, 15, 46546, 22737, 32783, 65331] avg=32768.0000 max=92051 max_rank=1 min=15 min_rank=3 imbalance_ratio=2.809174 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=32 tokens_per_ep_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] avg=32768.0000 max=33096 max_rank=3 min=32454 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65212 max_load_reduction_pct=0.663344 moved_to_replicas=97634 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=29 tokens_per_ep_rank=[2640, 92051, 41, 15, 46546, 22737, 32783, 65331] avg=32768.0000 max=92051 max_rank=1 min=15 min_rank=3 imbalance_ratio=2.809174 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=31 tokens_per_ep_rank=[54, 98265, 112, 18, 21458, 32748, 43773, 65716] avg=32768.0000 max=98265 max_rank=1 min=18 min_rank=3 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=31 tokens_per_ep_rank=[54, 98265, 112, 18, 21458, 32748, 43773, 65716] avg=32768.0000 max=98265 max_rank=1 min=18 min_rank=3 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=31 tokens_per_ep_rank=[54, 98265, 112, 18, 21458, 32748, 43773, 65716] avg=32768.0000 max=98265 max_rank=1 min=18 min_rank=3 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=29 tokens_per_ep_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] avg=32768.0000 max=33097 max_rank=5 min=31117 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=58954 max_load_reduction_pct=0.640449 moved_to_replicas=104640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=32 tokens_per_ep_rank=[32, 18, 32895, 65518, 32611, 98308, 12, 32750] avg=32768.0000 max=98308 max_rank=5 min=12 min_rank=6 imbalance_ratio=3.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=29 tokens_per_ep_rank=[33096, 33096, 31117, 32764, 33095, 33097, 32783, 33096] avg=32768.0000 max=33097 max_rank=5 min=31117 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=58954 max_load_reduction_pct=0.640449 moved_to_replicas=104640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=31 tokens_per_ep_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65169 max_load_reduction_pct=0.663196 moved_to_replicas=108466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=31 tokens_per_ep_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65169 max_load_reduction_pct=0.663196 moved_to_replicas=108466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=32 tokens_per_ep_rank=[32, 18, 32895, 65518, 32611, 98308, 12, 32750] avg=32768.0000 max=98308 max_rank=5 min=12 min_rank=6 imbalance_ratio=3.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=33 tokens_per_ep_rank=[21, 66598, 65474, 63, 32694, 197, 64322, 32775] avg=32768.0000 max=66598 max_rank=1 min=21 min_rank=0 imbalance_ratio=2.032410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=31 tokens_per_ep_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65169 max_load_reduction_pct=0.663196 moved_to_replicas=108466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=32 tokens_per_ep_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] avg=32768.0000 max=33096 max_rank=3 min=32454 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65212 max_load_reduction_pct=0.663344 moved_to_replicas=97634 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=32 tokens_per_ep_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] avg=32768.0000 max=33096 max_rank=3 min=32454 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65212 max_load_reduction_pct=0.663344 moved_to_replicas=97634 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=30 tokens_per_ep_rank=[32768, 67302, 18, 32796, 32777, 33983, 33025, 29475] avg=32768.0000 max=67302 max_rank=1 min=18 min_rank=2 imbalance_ratio=2.053894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=33 tokens_per_ep_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] avg=32768.0000 max=33096 max_rank=1 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33502 max_load_reduction_pct=0.503048 moved_to_replicas=97106 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=30 tokens_per_ep_rank=[32768, 67302, 18, 32796, 32777, 33983, 33025, 29475] avg=32768.0000 max=67302 max_rank=1 min=18 min_rank=2 imbalance_ratio=2.053894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=32 tokens_per_ep_rank=[32, 18, 32895, 65518, 32611, 98308, 12, 32750] avg=32768.0000 max=98308 max_rank=5 min=12 min_rank=6 imbalance_ratio=3.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=32 tokens_per_ep_rank=[32, 18, 32895, 65518, 32611, 98308, 12, 32750] avg=32768.0000 max=98308 max_rank=5 min=12 min_rank=6 imbalance_ratio=3.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=32 tokens_per_ep_rank=[32, 18, 32895, 65518, 32611, 98308, 12, 32750] avg=32768.0000 max=98308 max_rank=5 min=12 min_rank=6 imbalance_ratio=3.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=30 tokens_per_ep_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34206 max_load_reduction_pct=0.508246 moved_to_replicas=35093 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=33 tokens_per_ep_rank=[21, 66598, 65474, 63, 32694, 197, 64322, 32775] avg=32768.0000 max=66598 max_rank=1 min=21 min_rank=0 imbalance_ratio=2.032410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=33 tokens_per_ep_rank=[21, 66598, 65474, 63, 32694, 197, 64322, 32775] avg=32768.0000 max=66598 max_rank=1 min=21 min_rank=0 imbalance_ratio=2.032410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=34 tokens_per_ep_rank=[62890, 70053, 141, 18, 30762, 1, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=30 tokens_per_ep_rank=[32768, 33096, 32767, 32796, 32777, 33096, 33025, 31819] avg=32768.0000 max=33096 max_rank=1 min=31819 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34206 max_load_reduction_pct=0.508246 moved_to_replicas=35093 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=32 tokens_per_ep_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] avg=32768.0000 max=33096 max_rank=3 min=32454 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65212 max_load_reduction_pct=0.663344 moved_to_replicas=97634 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=32 tokens_per_ep_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] avg=32768.0000 max=33096 max_rank=3 min=32454 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65212 max_load_reduction_pct=0.663344 moved_to_replicas=97634 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=32 tokens_per_ep_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] avg=32768.0000 max=33096 max_rank=3 min=32454 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65212 max_load_reduction_pct=0.663344 moved_to_replicas=97634 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=33 tokens_per_ep_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] avg=32768.0000 max=33096 max_rank=1 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33502 max_load_reduction_pct=0.503048 moved_to_replicas=97106 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=31 tokens_per_ep_rank=[54, 98265, 112, 18, 21458, 32748, 43773, 65716] avg=32768.0000 max=98265 max_rank=1 min=18 min_rank=3 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=33 tokens_per_ep_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] avg=32768.0000 max=33096 max_rank=1 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33502 max_load_reduction_pct=0.503048 moved_to_replicas=97106 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=31815 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=131934 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=31 tokens_per_ep_rank=[54, 98265, 112, 18, 21458, 32748, 43773, 65716] avg=32768.0000 max=98265 max_rank=1 min=18 min_rank=3 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=33 tokens_per_ep_rank=[21, 66598, 65474, 63, 32694, 197, 64322, 32775] avg=32768.0000 max=66598 max_rank=1 min=21 min_rank=0 imbalance_ratio=2.032410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=33 tokens_per_ep_rank=[21, 66598, 65474, 63, 32694, 197, 64322, 32775] avg=32768.0000 max=66598 max_rank=1 min=21 min_rank=0 imbalance_ratio=2.032410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=33 tokens_per_ep_rank=[21, 66598, 65474, 63, 32694, 197, 64322, 32775] avg=32768.0000 max=66598 max_rank=1 min=21 min_rank=0 imbalance_ratio=2.032410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=34 tokens_per_ep_rank=[62890, 70053, 141, 18, 30762, 1, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=31 tokens_per_ep_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65169 max_load_reduction_pct=0.663196 moved_to_replicas=108466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=34 tokens_per_ep_rank=[62890, 70053, 141, 18, 30762, 1, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=35 tokens_per_ep_rank=[38, 21, 70, 111633, 32750, 52148, 65484, 0] avg=32768.0000 max=111633 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.406769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=31 tokens_per_ep_rank=[32474, 33096, 32732, 32767, 32136, 32748, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32136 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65169 max_load_reduction_pct=0.663196 moved_to_replicas=108466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=33 tokens_per_ep_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] avg=32768.0000 max=33096 max_rank=1 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33502 max_load_reduction_pct=0.503048 moved_to_replicas=97106 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=33 tokens_per_ep_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] avg=32768.0000 max=33096 max_rank=1 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33502 max_load_reduction_pct=0.503048 moved_to_replicas=97106 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=33 tokens_per_ep_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] avg=32768.0000 max=33096 max_rank=1 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33502 max_load_reduction_pct=0.503048 moved_to_replicas=97106 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=31815 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=131934 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=31815 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=131934 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=32 tokens_per_ep_rank=[32, 18, 32895, 65518, 32611, 98308, 12, 32750] avg=32768.0000 max=98308 max_rank=5 min=12 min_rank=6 imbalance_ratio=3.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=35 tokens_per_ep_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] avg=32768.0000 max=33096 max_rank=6 min=32131 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=78537 max_load_reduction_pct=0.703529 moved_to_replicas=129977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=32 tokens_per_ep_rank=[32, 18, 32895, 65518, 32611, 98308, 12, 32750] avg=32768.0000 max=98308 max_rank=5 min=12 min_rank=6 imbalance_ratio=3.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=34 tokens_per_ep_rank=[62890, 70053, 141, 18, 30762, 1, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=34 tokens_per_ep_rank=[62890, 70053, 141, 18, 30762, 1, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=34 tokens_per_ep_rank=[62890, 70053, 141, 18, 30762, 1, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=35 tokens_per_ep_rank=[38, 21, 70, 111633, 32750, 52148, 65484, 0] avg=32768.0000 max=111633 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.406769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=32 tokens_per_ep_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] avg=32768.0000 max=33096 max_rank=3 min=32454 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65212 max_load_reduction_pct=0.663344 moved_to_replicas=97634 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=35 tokens_per_ep_rank=[38, 21, 70, 111633, 32750, 52148, 65484, 0] avg=32768.0000 max=111633 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.406769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=36 tokens_per_ep_rank=[98585, 32430, 19, 98176, 114, 32793, 24, 3] avg=32768.0000 max=98585 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=32 tokens_per_ep_rank=[32454, 32481, 32895, 33096, 32611, 33096, 32761, 32750] avg=32768.0000 max=33096 max_rank=3 min=32454 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65212 max_load_reduction_pct=0.663344 moved_to_replicas=97634 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=31815 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=131934 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=35 tokens_per_ep_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] avg=32768.0000 max=33096 max_rank=6 min=32131 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=78537 max_load_reduction_pct=0.703529 moved_to_replicas=129977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=31815 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=131934 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=31815 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=131934 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=35 tokens_per_ep_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] avg=32768.0000 max=33096 max_rank=6 min=32131 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=78537 max_load_reduction_pct=0.703529 moved_to_replicas=129977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=33 tokens_per_ep_rank=[21, 66598, 65474, 63, 32694, 197, 64322, 32775] avg=32768.0000 max=66598 max_rank=1 min=21 min_rank=0 imbalance_ratio=2.032410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=36 tokens_per_ep_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65489 max_load_reduction_pct=0.664290 moved_to_replicas=130569 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=33 tokens_per_ep_rank=[21, 66598, 65474, 63, 32694, 197, 64322, 32775] avg=32768.0000 max=66598 max_rank=1 min=21 min_rank=0 imbalance_ratio=2.032410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=35 tokens_per_ep_rank=[38, 21, 70, 111633, 32750, 52148, 65484, 0] avg=32768.0000 max=111633 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.406769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=36 tokens_per_ep_rank=[98585, 32430, 19, 98176, 114, 32793, 24, 3] avg=32768.0000 max=98585 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=35 tokens_per_ep_rank=[38, 21, 70, 111633, 32750, 52148, 65484, 0] avg=32768.0000 max=111633 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.406769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=35 tokens_per_ep_rank=[38, 21, 70, 111633, 32750, 52148, 65484, 0] avg=32768.0000 max=111633 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.406769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=33 tokens_per_ep_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] avg=32768.0000 max=33096 max_rank=1 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33502 max_load_reduction_pct=0.503048 moved_to_replicas=97106 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=36 tokens_per_ep_rank=[98585, 32430, 19, 98176, 114, 32793, 24, 3] avg=32768.0000 max=98585 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=37 tokens_per_ep_rank=[62039, 370, 54, 0, 33199, 0, 68622, 97860] avg=32768.0000 max=97860 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.986450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=33 tokens_per_ep_rank=[32761, 33096, 33096, 32051, 32694, 32575, 33096, 32775] avg=32768.0000 max=33096 max_rank=1 min=32051 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33502 max_load_reduction_pct=0.503048 moved_to_replicas=97106 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=35 tokens_per_ep_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] avg=32768.0000 max=33096 max_rank=6 min=32131 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=78537 max_load_reduction_pct=0.703529 moved_to_replicas=129977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=36 tokens_per_ep_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65489 max_load_reduction_pct=0.664290 moved_to_replicas=130569 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=35 tokens_per_ep_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] avg=32768.0000 max=33096 max_rank=6 min=32131 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=78537 max_load_reduction_pct=0.703529 moved_to_replicas=129977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=35 tokens_per_ep_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] avg=32768.0000 max=33096 max_rank=6 min=32131 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=78537 max_load_reduction_pct=0.703529 moved_to_replicas=129977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=36 tokens_per_ep_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65489 max_load_reduction_pct=0.664290 moved_to_replicas=130569 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=37 tokens_per_ep_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32090 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64764 max_load_reduction_pct=0.661803 moved_to_replicas=129336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=34 tokens_per_ep_rank=[62890, 70053, 141, 18, 30762, 1, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=34 tokens_per_ep_rank=[62890, 70053, 141, 18, 30762, 1, 0, 98279] avg=32768.0000 max=98279 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=36 tokens_per_ep_rank=[98585, 32430, 19, 98176, 114, 32793, 24, 3] avg=32768.0000 max=98585 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=37 tokens_per_ep_rank=[62039, 370, 54, 0, 33199, 0, 68622, 97860] avg=32768.0000 max=97860 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.986450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=36 tokens_per_ep_rank=[98585, 32430, 19, 98176, 114, 32793, 24, 3] avg=32768.0000 max=98585 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=36 tokens_per_ep_rank=[98585, 32430, 19, 98176, 114, 32793, 24, 3] avg=32768.0000 max=98585 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=31815 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=131934 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=37 tokens_per_ep_rank=[62039, 370, 54, 0, 33199, 0, 68622, 97860] avg=32768.0000 max=97860 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.986450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=38 tokens_per_ep_rank=[25072, 14, 33212, 32770, 15604, 56424, 32740, 66308] avg=32768.0000 max=66308 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.023560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32761, 31815, 32435, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=31815 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65183 max_load_reduction_pct=0.663244 moved_to_replicas=131934 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=36 tokens_per_ep_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65489 max_load_reduction_pct=0.664290 moved_to_replicas=130569 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=37 tokens_per_ep_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32090 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64764 max_load_reduction_pct=0.661803 moved_to_replicas=129336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=36 tokens_per_ep_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65489 max_load_reduction_pct=0.664290 moved_to_replicas=130569 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=36 tokens_per_ep_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65489 max_load_reduction_pct=0.664290 moved_to_replicas=130569 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=37 tokens_per_ep_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32090 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64764 max_load_reduction_pct=0.661803 moved_to_replicas=129336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=38 tokens_per_ep_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] avg=32768.0000 max=33097 max_rank=5 min=31487 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33211 max_load_reduction_pct=0.500860 moved_to_replicas=56656 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=35 tokens_per_ep_rank=[38, 21, 70, 111633, 32750, 52148, 65484, 0] avg=32768.0000 max=111633 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.406769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=37 tokens_per_ep_rank=[62039, 370, 54, 0, 33199, 0, 68622, 97860] avg=32768.0000 max=97860 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.986450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=35 tokens_per_ep_rank=[38, 21, 70, 111633, 32750, 52148, 65484, 0] avg=32768.0000 max=111633 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.406769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=38 tokens_per_ep_rank=[25072, 14, 33212, 32770, 15604, 56424, 32740, 66308] avg=32768.0000 max=66308 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.023560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=37 tokens_per_ep_rank=[62039, 370, 54, 0, 33199, 0, 68622, 97860] avg=32768.0000 max=97860 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.986450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=37 tokens_per_ep_rank=[62039, 370, 54, 0, 33199, 0, 68622, 97860] avg=32768.0000 max=97860 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.986450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=38 tokens_per_ep_rank=[25072, 14, 33212, 32770, 15604, 56424, 32740, 66308] avg=32768.0000 max=66308 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.023560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=35 tokens_per_ep_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] avg=32768.0000 max=33096 max_rank=6 min=32131 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=78537 max_load_reduction_pct=0.703529 moved_to_replicas=129977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=39 tokens_per_ep_rank=[55996, 0, 32815, 32803, 36, 71441, 36506, 32547] avg=32768.0000 max=71441 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.180206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=37 tokens_per_ep_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32090 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64764 max_load_reduction_pct=0.661803 moved_to_replicas=129336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=35 tokens_per_ep_rank=[32131, 32770, 32458, 33095, 32750, 33095, 33096, 32749] avg=32768.0000 max=33096 max_rank=6 min=32131 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=78537 max_load_reduction_pct=0.703529 moved_to_replicas=129977 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=38 tokens_per_ep_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] avg=32768.0000 max=33097 max_rank=5 min=31487 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33211 max_load_reduction_pct=0.500860 moved_to_replicas=56656 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=37 tokens_per_ep_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32090 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64764 max_load_reduction_pct=0.661803 moved_to_replicas=129336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=37 tokens_per_ep_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32090 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64764 max_load_reduction_pct=0.661803 moved_to_replicas=129336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=38 tokens_per_ep_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] avg=32768.0000 max=33097 max_rank=5 min=31487 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33211 max_load_reduction_pct=0.500860 moved_to_replicas=56656 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] avg=32768.0000 max=33096 max_rank=0 min=31942 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=38345 max_load_reduction_pct=0.536737 moved_to_replicas=64655 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=36 tokens_per_ep_rank=[98585, 32430, 19, 98176, 114, 32793, 24, 3] avg=32768.0000 max=98585 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=38 tokens_per_ep_rank=[25072, 14, 33212, 32770, 15604, 56424, 32740, 66308] avg=32768.0000 max=66308 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.023560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=36 tokens_per_ep_rank=[98585, 32430, 19, 98176, 114, 32793, 24, 3] avg=32768.0000 max=98585 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=39 tokens_per_ep_rank=[55996, 0, 32815, 32803, 36, 71441, 36506, 32547] avg=32768.0000 max=71441 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.180206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=38 tokens_per_ep_rank=[25072, 14, 33212, 32770, 15604, 56424, 32740, 66308] avg=32768.0000 max=66308 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.023560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=38 tokens_per_ep_rank=[25072, 14, 33212, 32770, 15604, 56424, 32740, 66308] avg=32768.0000 max=66308 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.023560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=39 tokens_per_ep_rank=[55996, 0, 32815, 32803, 36, 71441, 36506, 32547] avg=32768.0000 max=71441 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.180206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=40 tokens_per_ep_rank=[65540, 74974, 5895, 32779, 40, 32744, 50154, 18] avg=32768.0000 max=74974 max_rank=1 min=18 min_rank=7 imbalance_ratio=2.288025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=36 tokens_per_ep_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65489 max_load_reduction_pct=0.664290 moved_to_replicas=130569 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=38 tokens_per_ep_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] avg=32768.0000 max=33097 max_rank=5 min=31487 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33211 max_load_reduction_pct=0.500860 moved_to_replicas=56656 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=36 tokens_per_ep_rank=[33096, 32430, 32759, 33096, 32445, 32793, 32773, 32752] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65489 max_load_reduction_pct=0.664290 moved_to_replicas=130569 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] avg=32768.0000 max=33096 max_rank=0 min=31942 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=38345 max_load_reduction_pct=0.536737 moved_to_replicas=64655 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=38 tokens_per_ep_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] avg=32768.0000 max=33097 max_rank=5 min=31487 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33211 max_load_reduction_pct=0.500860 moved_to_replicas=56656 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=38 tokens_per_ep_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] avg=32768.0000 max=33097 max_rank=5 min=31487 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33211 max_load_reduction_pct=0.500860 moved_to_replicas=56656 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] avg=32768.0000 max=33096 max_rank=0 min=31942 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=38345 max_load_reduction_pct=0.536737 moved_to_replicas=64655 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=40 tokens_per_ep_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] avg=32768.0000 max=33097 max_rank=1 min=31470 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41877 max_load_reduction_pct=0.558554 moved_to_replicas=91380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=39 tokens_per_ep_rank=[55996, 0, 32815, 32803, 36, 71441, 36506, 32547] avg=32768.0000 max=71441 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.180206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=37 tokens_per_ep_rank=[62039, 370, 54, 0, 33199, 0, 68622, 97860] avg=32768.0000 max=97860 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.986450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=37 tokens_per_ep_rank=[62039, 370, 54, 0, 33199, 0, 68622, 97860] avg=32768.0000 max=97860 max_rank=7 min=0 min_rank=3 imbalance_ratio=2.986450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=40 tokens_per_ep_rank=[65540, 74974, 5895, 32779, 40, 32744, 50154, 18] avg=32768.0000 max=74974 max_rank=1 min=18 min_rank=7 imbalance_ratio=2.288025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=39 tokens_per_ep_rank=[55996, 0, 32815, 32803, 36, 71441, 36506, 32547] avg=32768.0000 max=71441 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.180206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=40 tokens_per_ep_rank=[65540, 74974, 5895, 32779, 40, 32744, 50154, 18] avg=32768.0000 max=74974 max_rank=1 min=18 min_rank=7 imbalance_ratio=2.288025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=39 tokens_per_ep_rank=[55996, 0, 32815, 32803, 36, 71441, 36506, 32547] avg=32768.0000 max=71441 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.180206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=41 tokens_per_ep_rank=[18, 18, 64561, 65603, 21, 47858, 32672, 51393] avg=32768.0000 max=65603 max_rank=3 min=18 min_rank=0 imbalance_ratio=2.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] avg=32768.0000 max=33096 max_rank=0 min=31942 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=38345 max_load_reduction_pct=0.536737 moved_to_replicas=64655 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=37 tokens_per_ep_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32090 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64764 max_load_reduction_pct=0.661803 moved_to_replicas=129336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=37 tokens_per_ep_rank=[33096, 32090, 32803, 32749, 33096, 32118, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32090 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64764 max_load_reduction_pct=0.661803 moved_to_replicas=129336 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=40 tokens_per_ep_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] avg=32768.0000 max=33097 max_rank=1 min=31470 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41877 max_load_reduction_pct=0.558554 moved_to_replicas=91380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] avg=32768.0000 max=33096 max_rank=0 min=31942 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=38345 max_load_reduction_pct=0.536737 moved_to_replicas=64655 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=40 tokens_per_ep_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] avg=32768.0000 max=33097 max_rank=1 min=31470 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41877 max_load_reduction_pct=0.558554 moved_to_replicas=91380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] avg=32768.0000 max=33096 max_rank=0 min=31942 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=38345 max_load_reduction_pct=0.536737 moved_to_replicas=64655 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=41 tokens_per_ep_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] avg=32768.0000 max=33097 max_rank=7 min=31483 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32506 max_load_reduction_pct=0.495496 moved_to_replicas=97031 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=40 tokens_per_ep_rank=[65540, 74974, 5895, 32779, 40, 32744, 50154, 18] avg=32768.0000 max=74974 max_rank=1 min=18 min_rank=7 imbalance_ratio=2.288025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=38 tokens_per_ep_rank=[25072, 14, 33212, 32770, 15604, 56424, 32740, 66308] avg=32768.0000 max=66308 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.023560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=41 tokens_per_ep_rank=[18, 18, 64561, 65603, 21, 47858, 32672, 51393] avg=32768.0000 max=65603 max_rank=3 min=18 min_rank=0 imbalance_ratio=2.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=38 tokens_per_ep_rank=[25072, 14, 33212, 32770, 15604, 56424, 32740, 66308] avg=32768.0000 max=66308 max_rank=7 min=14 min_rank=1 imbalance_ratio=2.023560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=40 tokens_per_ep_rank=[65540, 74974, 5895, 32779, 40, 32744, 50154, 18] avg=32768.0000 max=74974 max_rank=1 min=18 min_rank=7 imbalance_ratio=2.288025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=41 tokens_per_ep_rank=[18, 18, 64561, 65603, 21, 47858, 32672, 51393] avg=32768.0000 max=65603 max_rank=3 min=18 min_rank=0 imbalance_ratio=2.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=40 tokens_per_ep_rank=[65540, 74974, 5895, 32779, 40, 32744, 50154, 18] avg=32768.0000 max=74974 max_rank=1 min=18 min_rank=7 imbalance_ratio=2.288025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=42 tokens_per_ep_rank=[37635, 147, 4459, 65152, 56523, 65496, 32728, 4] avg=32768.0000 max=65496 max_rank=5 min=4 min_rank=7 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=40 tokens_per_ep_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] avg=32768.0000 max=33097 max_rank=1 min=31470 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41877 max_load_reduction_pct=0.558554 moved_to_replicas=91380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=38 tokens_per_ep_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] avg=32768.0000 max=33097 max_rank=5 min=31487 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33211 max_load_reduction_pct=0.500860 moved_to_replicas=56656 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=41 tokens_per_ep_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] avg=32768.0000 max=33097 max_rank=7 min=31483 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32506 max_load_reduction_pct=0.495496 moved_to_replicas=97031 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=38 tokens_per_ep_rank=[31487, 32762, 33096, 32770, 33096, 33097, 32740, 33096] avg=32768.0000 max=33097 max_rank=5 min=31487 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=33211 max_load_reduction_pct=0.500860 moved_to_replicas=56656 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=41 tokens_per_ep_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] avg=32768.0000 max=33097 max_rank=7 min=31483 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32506 max_load_reduction_pct=0.495496 moved_to_replicas=97031 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=40 tokens_per_ep_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] avg=32768.0000 max=33097 max_rank=1 min=31470 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41877 max_load_reduction_pct=0.558554 moved_to_replicas=91380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=40 tokens_per_ep_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] avg=32768.0000 max=33097 max_rank=1 min=31470 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41877 max_load_reduction_pct=0.558554 moved_to_replicas=91380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=42 tokens_per_ep_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] avg=32768.0000 max=33096 max_rank=0 min=32203 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=92422 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=41 tokens_per_ep_rank=[18, 18, 64561, 65603, 21, 47858, 32672, 51393] avg=32768.0000 max=65603 max_rank=3 min=18 min_rank=0 imbalance_ratio=2.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=39 tokens_per_ep_rank=[55996, 0, 32815, 32803, 36, 71441, 36506, 32547] avg=32768.0000 max=71441 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.180206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=42 tokens_per_ep_rank=[37635, 147, 4459, 65152, 56523, 65496, 32728, 4] avg=32768.0000 max=65496 max_rank=5 min=4 min_rank=7 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=39 tokens_per_ep_rank=[55996, 0, 32815, 32803, 36, 71441, 36506, 32547] avg=32768.0000 max=71441 max_rank=5 min=0 min_rank=1 imbalance_ratio=2.180206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=42 tokens_per_ep_rank=[37635, 147, 4459, 65152, 56523, 65496, 32728, 4] avg=32768.0000 max=65496 max_rank=5 min=4 min_rank=7 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=41 tokens_per_ep_rank=[18, 18, 64561, 65603, 21, 47858, 32672, 51393] avg=32768.0000 max=65603 max_rank=3 min=18 min_rank=0 imbalance_ratio=2.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=41 tokens_per_ep_rank=[18, 18, 64561, 65603, 21, 47858, 32672, 51393] avg=32768.0000 max=65603 max_rank=3 min=18 min_rank=0 imbalance_ratio=2.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=41 tokens_per_ep_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] avg=32768.0000 max=33097 max_rank=7 min=31483 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32506 max_load_reduction_pct=0.495496 moved_to_replicas=97031 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 295, 1057, 32242, 32827, 45578, 84674] avg=32768.0000 max=84674 max_rank=7 min=295 min_rank=2 imbalance_ratio=2.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] avg=32768.0000 max=33096 max_rank=0 min=31942 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=38345 max_load_reduction_pct=0.536737 moved_to_replicas=64655 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=42 tokens_per_ep_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] avg=32768.0000 max=33096 max_rank=0 min=32203 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=92422 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32749, 32815, 32803, 31942, 33096, 33096, 32547] avg=32768.0000 max=33096 max_rank=0 min=31942 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=38345 max_load_reduction_pct=0.536737 moved_to_replicas=64655 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=42 tokens_per_ep_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] avg=32768.0000 max=33096 max_rank=0 min=32203 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=92422 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=41 tokens_per_ep_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] avg=32768.0000 max=33097 max_rank=7 min=31483 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32506 max_load_reduction_pct=0.495496 moved_to_replicas=97031 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=41 tokens_per_ep_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] avg=32768.0000 max=33097 max_rank=7 min=31483 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32506 max_load_reduction_pct=0.495496 moved_to_replicas=97031 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] avg=32768.0000 max=33095 max_rank=6 min=32242 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=51579 max_load_reduction_pct=0.609148 moved_to_replicas=64060 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=42 tokens_per_ep_rank=[37635, 147, 4459, 65152, 56523, 65496, 32728, 4] avg=32768.0000 max=65496 max_rank=5 min=4 min_rank=7 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=40 tokens_per_ep_rank=[65540, 74974, 5895, 32779, 40, 32744, 50154, 18] avg=32768.0000 max=74974 max_rank=1 min=18 min_rank=7 imbalance_ratio=2.288025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 295, 1057, 32242, 32827, 45578, 84674] avg=32768.0000 max=84674 max_rank=7 min=295 min_rank=2 imbalance_ratio=2.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=40 tokens_per_ep_rank=[65540, 74974, 5895, 32779, 40, 32744, 50154, 18] avg=32768.0000 max=74974 max_rank=1 min=18 min_rank=7 imbalance_ratio=2.288025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 295, 1057, 32242, 32827, 45578, 84674] avg=32768.0000 max=84674 max_rank=7 min=295 min_rank=2 imbalance_ratio=2.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=42 tokens_per_ep_rank=[37635, 147, 4459, 65152, 56523, 65496, 32728, 4] avg=32768.0000 max=65496 max_rank=5 min=4 min_rank=7 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=42 tokens_per_ep_rank=[37635, 147, 4459, 65152, 56523, 65496, 32728, 4] avg=32768.0000 max=65496 max_rank=5 min=4 min_rank=7 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=42 tokens_per_ep_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] avg=32768.0000 max=33096 max_rank=0 min=32203 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=92422 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=44 tokens_per_ep_rank=[1, 64, 146644, 35334, 177, 7699, 39402, 32823] avg=32768.0000 max=146644 max_rank=2 min=1 min_rank=0 imbalance_ratio=4.475220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=40 tokens_per_ep_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] avg=32768.0000 max=33097 max_rank=1 min=31470 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41877 max_load_reduction_pct=0.558554 moved_to_replicas=91380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] avg=32768.0000 max=33095 max_rank=6 min=32242 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=51579 max_load_reduction_pct=0.609148 moved_to_replicas=64060 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=40 tokens_per_ep_rank=[33096, 33097, 33096, 32779, 31470, 32744, 33095, 32767] avg=32768.0000 max=33097 max_rank=1 min=31470 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41877 max_load_reduction_pct=0.558554 moved_to_replicas=91380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] avg=32768.0000 max=33095 max_rank=6 min=32242 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=51579 max_load_reduction_pct=0.609148 moved_to_replicas=64060 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=42 tokens_per_ep_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] avg=32768.0000 max=33096 max_rank=0 min=32203 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=92422 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=42 tokens_per_ep_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] avg=32768.0000 max=33096 max_rank=0 min=32203 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=92422 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=44 tokens_per_ep_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] avg=32768.0000 max=33096 max_rank=2 min=31635 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=113548 max_load_reduction_pct=0.774311 moved_to_replicas=122092 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 295, 1057, 32242, 32827, 45578, 84674] avg=32768.0000 max=84674 max_rank=7 min=295 min_rank=2 imbalance_ratio=2.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=41 tokens_per_ep_rank=[18, 18, 64561, 65603, 21, 47858, 32672, 51393] avg=32768.0000 max=65603 max_rank=3 min=18 min_rank=0 imbalance_ratio=2.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=44 tokens_per_ep_rank=[1, 64, 146644, 35334, 177, 7699, 39402, 32823] avg=32768.0000 max=146644 max_rank=2 min=1 min_rank=0 imbalance_ratio=4.475220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=41 tokens_per_ep_rank=[18, 18, 64561, 65603, 21, 47858, 32672, 51393] avg=32768.0000 max=65603 max_rank=3 min=18 min_rank=0 imbalance_ratio=2.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=44 tokens_per_ep_rank=[1, 64, 146644, 35334, 177, 7699, 39402, 32823] avg=32768.0000 max=146644 max_rank=2 min=1 min_rank=0 imbalance_ratio=4.475220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 295, 1057, 32242, 32827, 45578, 84674] avg=32768.0000 max=84674 max_rank=7 min=295 min_rank=2 imbalance_ratio=2.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 295, 1057, 32242, 32827, 45578, 84674] avg=32768.0000 max=84674 max_rank=7 min=295 min_rank=2 imbalance_ratio=2.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] avg=32768.0000 max=33095 max_rank=6 min=32242 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=51579 max_load_reduction_pct=0.609148 moved_to_replicas=64060 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=45 tokens_per_ep_rank=[32769, 65518, 98250, 0, 32786, 16, 19, 32786] avg=32768.0000 max=98250 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=41 tokens_per_ep_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] avg=32768.0000 max=33097 max_rank=7 min=31483 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32506 max_load_reduction_pct=0.495496 moved_to_replicas=97031 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=44 tokens_per_ep_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] avg=32768.0000 max=33096 max_rank=2 min=31635 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=113548 max_load_reduction_pct=0.774311 moved_to_replicas=122092 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=44 tokens_per_ep_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] avg=32768.0000 max=33096 max_rank=2 min=31635 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=113548 max_load_reduction_pct=0.774311 moved_to_replicas=122092 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] avg=32768.0000 max=33095 max_rank=6 min=32242 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=51579 max_load_reduction_pct=0.609148 moved_to_replicas=64060 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=41 tokens_per_ep_rank=[32525, 31483, 33096, 33096, 33079, 33096, 32672, 33097] avg=32768.0000 max=33097 max_rank=7 min=31483 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=32506 max_load_reduction_pct=0.495496 moved_to_replicas=97031 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] avg=32768.0000 max=33095 max_rank=6 min=32242 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=51579 max_load_reduction_pct=0.609148 moved_to_replicas=64060 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=45 tokens_per_ep_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65154 max_load_reduction_pct=0.663145 moved_to_replicas=97576 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=44 tokens_per_ep_rank=[1, 64, 146644, 35334, 177, 7699, 39402, 32823] avg=32768.0000 max=146644 max_rank=2 min=1 min_rank=0 imbalance_ratio=4.475220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=45 tokens_per_ep_rank=[32769, 65518, 98250, 0, 32786, 16, 19, 32786] avg=32768.0000 max=98250 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=42 tokens_per_ep_rank=[37635, 147, 4459, 65152, 56523, 65496, 32728, 4] avg=32768.0000 max=65496 max_rank=5 min=4 min_rank=7 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=45 tokens_per_ep_rank=[32769, 65518, 98250, 0, 32786, 16, 19, 32786] avg=32768.0000 max=98250 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=44 tokens_per_ep_rank=[1, 64, 146644, 35334, 177, 7699, 39402, 32823] avg=32768.0000 max=146644 max_rank=2 min=1 min_rank=0 imbalance_ratio=4.475220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=42 tokens_per_ep_rank=[37635, 147, 4459, 65152, 56523, 65496, 32728, 4] avg=32768.0000 max=65496 max_rank=5 min=4 min_rank=7 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=44 tokens_per_ep_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] avg=32768.0000 max=33096 max_rank=2 min=31635 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=113548 max_load_reduction_pct=0.774311 moved_to_replicas=122092 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=44 tokens_per_ep_rank=[1, 64, 146644, 35334, 177, 7699, 39402, 32823] avg=32768.0000 max=146644 max_rank=2 min=1 min_rank=0 imbalance_ratio=4.475220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=46 tokens_per_ep_rank=[32776, 37, 33606, 32750, 35, 65106, 32768, 65066] avg=32768.0000 max=65106 max_rank=5 min=35 min_rank=4 imbalance_ratio=1.986877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=45 tokens_per_ep_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65154 max_load_reduction_pct=0.663145 moved_to_replicas=97576 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=42 tokens_per_ep_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] avg=32768.0000 max=33096 max_rank=0 min=32203 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=92422 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=45 tokens_per_ep_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65154 max_load_reduction_pct=0.663145 moved_to_replicas=97576 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=44 tokens_per_ep_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] avg=32768.0000 max=33096 max_rank=2 min=31635 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=113548 max_load_reduction_pct=0.774311 moved_to_replicas=122092 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=42 tokens_per_ep_rank=[33096, 32203, 32426, 33096, 33095, 33096, 32728, 32404] avg=32768.0000 max=33096 max_rank=0 min=32203 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=92422 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=46 tokens_per_ep_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32010 max_load_reduction_pct=0.491660 moved_to_replicas=64490 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=44 tokens_per_ep_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] avg=32768.0000 max=33096 max_rank=2 min=31635 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=113548 max_load_reduction_pct=0.774311 moved_to_replicas=122092 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=45 tokens_per_ep_rank=[32769, 65518, 98250, 0, 32786, 16, 19, 32786] avg=32768.0000 max=98250 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=46 tokens_per_ep_rank=[32776, 37, 33606, 32750, 35, 65106, 32768, 65066] avg=32768.0000 max=65106 max_rank=5 min=35 min_rank=4 imbalance_ratio=1.986877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 295, 1057, 32242, 32827, 45578, 84674] avg=32768.0000 max=84674 max_rank=7 min=295 min_rank=2 imbalance_ratio=2.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=46 tokens_per_ep_rank=[32776, 37, 33606, 32750, 35, 65106, 32768, 65066] avg=32768.0000 max=65106 max_rank=5 min=35 min_rank=4 imbalance_ratio=1.986877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=45 tokens_per_ep_rank=[32769, 65518, 98250, 0, 32786, 16, 19, 32786] avg=32768.0000 max=98250 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 295, 1057, 32242, 32827, 45578, 84674] avg=32768.0000 max=84674 max_rank=7 min=295 min_rank=2 imbalance_ratio=2.584045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=45 tokens_per_ep_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65154 max_load_reduction_pct=0.663145 moved_to_replicas=97576 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=47 tokens_per_ep_rank=[106, 1238, 32768, 68129, 32751, 32761, 32623, 61768] avg=32768.0000 max=68129 max_rank=3 min=106 min_rank=0 imbalance_ratio=2.079132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=45 tokens_per_ep_rank=[32769, 65518, 98250, 0, 32786, 16, 19, 32786] avg=32768.0000 max=98250 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=46 tokens_per_ep_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32010 max_load_reduction_pct=0.491660 moved_to_replicas=64490 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] avg=32768.0000 max=33095 max_rank=6 min=32242 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=51579 max_load_reduction_pct=0.609148 moved_to_replicas=64060 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=46 tokens_per_ep_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32010 max_load_reduction_pct=0.491660 moved_to_replicas=64490 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=45 tokens_per_ep_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65154 max_load_reduction_pct=0.663145 moved_to_replicas=97576 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=43 tokens_per_ep_rank=[32735, 32736, 33044, 32370, 32242, 32827, 33095, 33095] avg=32768.0000 max=33095 max_rank=6 min=32242 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=51579 max_load_reduction_pct=0.609148 moved_to_replicas=64060 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=47 tokens_per_ep_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] avg=32768.0000 max=33096 max_rank=3 min=32195 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35033 max_load_reduction_pct=0.514216 moved_to_replicas=63705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=45 tokens_per_ep_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65154 max_load_reduction_pct=0.663145 moved_to_replicas=97576 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=46 tokens_per_ep_rank=[32776, 37, 33606, 32750, 35, 65106, 32768, 65066] avg=32768.0000 max=65106 max_rank=5 min=35 min_rank=4 imbalance_ratio=1.986877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=47 tokens_per_ep_rank=[106, 1238, 32768, 68129, 32751, 32761, 32623, 61768] avg=32768.0000 max=68129 max_rank=3 min=106 min_rank=0 imbalance_ratio=2.079132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=47 tokens_per_ep_rank=[106, 1238, 32768, 68129, 32751, 32761, 32623, 61768] avg=32768.0000 max=68129 max_rank=3 min=106 min_rank=0 imbalance_ratio=2.079132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=44 tokens_per_ep_rank=[1, 64, 146644, 35334, 177, 7699, 39402, 32823] avg=32768.0000 max=146644 max_rank=2 min=1 min_rank=0 imbalance_ratio=4.475220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=46 tokens_per_ep_rank=[32776, 37, 33606, 32750, 35, 65106, 32768, 65066] avg=32768.0000 max=65106 max_rank=5 min=35 min_rank=4 imbalance_ratio=1.986877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=44 tokens_per_ep_rank=[1, 64, 146644, 35334, 177, 7699, 39402, 32823] avg=32768.0000 max=146644 max_rank=2 min=1 min_rank=0 imbalance_ratio=4.475220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=46 tokens_per_ep_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32010 max_load_reduction_pct=0.491660 moved_to_replicas=64490 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=48 tokens_per_ep_rank=[96544, 65500, 18, 32643, 2597, 59046, 2482, 3314] avg=32768.0000 max=96544 max_rank=0 min=18 min_rank=2 imbalance_ratio=2.946289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=46 tokens_per_ep_rank=[32776, 37, 33606, 32750, 35, 65106, 32768, 65066] avg=32768.0000 max=65106 max_rank=5 min=35 min_rank=4 imbalance_ratio=1.986877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=47 tokens_per_ep_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] avg=32768.0000 max=33096 max_rank=3 min=32195 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35033 max_load_reduction_pct=0.514216 moved_to_replicas=63705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=47 tokens_per_ep_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] avg=32768.0000 max=33096 max_rank=3 min=32195 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35033 max_load_reduction_pct=0.514216 moved_to_replicas=63705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=46 tokens_per_ep_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32010 max_load_reduction_pct=0.491660 moved_to_replicas=64490 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=44 tokens_per_ep_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] avg=32768.0000 max=33096 max_rank=2 min=31635 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=113548 max_load_reduction_pct=0.774311 moved_to_replicas=122092 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=44 tokens_per_ep_rank=[32750, 32795, 33096, 33096, 32854, 31635, 33095, 32823] avg=32768.0000 max=33096 max_rank=2 min=31635 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=113548 max_load_reduction_pct=0.774311 moved_to_replicas=122092 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] avg=32768.0000 max=33096 max_rank=0 min=31254 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63448 max_load_reduction_pct=0.657193 moved_to_replicas=121802 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=46 tokens_per_ep_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32010 max_load_reduction_pct=0.491660 moved_to_replicas=64490 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=47 tokens_per_ep_rank=[106, 1238, 32768, 68129, 32751, 32761, 32623, 61768] avg=32768.0000 max=68129 max_rank=3 min=106 min_rank=0 imbalance_ratio=2.079132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=48 tokens_per_ep_rank=[96544, 65500, 18, 32643, 2597, 59046, 2482, 3314] avg=32768.0000 max=96544 max_rank=0 min=18 min_rank=2 imbalance_ratio=2.946289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=48 tokens_per_ep_rank=[96544, 65500, 18, 32643, 2597, 59046, 2482, 3314] avg=32768.0000 max=96544 max_rank=0 min=18 min_rank=2 imbalance_ratio=2.946289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=47 tokens_per_ep_rank=[106, 1238, 32768, 68129, 32751, 32761, 32623, 61768] avg=32768.0000 max=68129 max_rank=3 min=106 min_rank=0 imbalance_ratio=2.079132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=45 tokens_per_ep_rank=[32769, 65518, 98250, 0, 32786, 16, 19, 32786] avg=32768.0000 max=98250 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=45 tokens_per_ep_rank=[32769, 65518, 98250, 0, 32786, 16, 19, 32786] avg=32768.0000 max=98250 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=47 tokens_per_ep_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] avg=32768.0000 max=33096 max_rank=3 min=32195 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35033 max_load_reduction_pct=0.514216 moved_to_replicas=63705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] avg=32768.0000 max=33096 max_rank=0 min=31254 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63448 max_load_reduction_pct=0.657193 moved_to_replicas=121802 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=47 tokens_per_ep_rank=[106, 1238, 32768, 68129, 32751, 32761, 32623, 61768] avg=32768.0000 max=68129 max_rank=3 min=106 min_rank=0 imbalance_ratio=2.079132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] avg=32768.0000 max=33096 max_rank=0 min=31254 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63448 max_load_reduction_pct=0.657193 moved_to_replicas=121802 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=47 tokens_per_ep_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] avg=32768.0000 max=33096 max_rank=3 min=32195 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35033 max_load_reduction_pct=0.514216 moved_to_replicas=63705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=45 tokens_per_ep_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65154 max_load_reduction_pct=0.663145 moved_to_replicas=97576 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=45 tokens_per_ep_rank=[32769, 33096, 33096, 32749, 32786, 32421, 32441, 32786] avg=32768.0000 max=33096 max_rank=1 min=32421 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65154 max_load_reduction_pct=0.663145 moved_to_replicas=97576 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=47 tokens_per_ep_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] avg=32768.0000 max=33096 max_rank=3 min=32195 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35033 max_load_reduction_pct=0.514216 moved_to_replicas=63705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=48 tokens_per_ep_rank=[96544, 65500, 18, 32643, 2597, 59046, 2482, 3314] avg=32768.0000 max=96544 max_rank=0 min=18 min_rank=2 imbalance_ratio=2.946289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=48 tokens_per_ep_rank=[96544, 65500, 18, 32643, 2597, 59046, 2482, 3314] avg=32768.0000 max=96544 max_rank=0 min=18 min_rank=2 imbalance_ratio=2.946289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=46 tokens_per_ep_rank=[32776, 37, 33606, 32750, 35, 65106, 32768, 65066] avg=32768.0000 max=65106 max_rank=5 min=35 min_rank=4 imbalance_ratio=1.986877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] avg=32768.0000 max=33096 max_rank=0 min=31254 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63448 max_load_reduction_pct=0.657193 moved_to_replicas=121802 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=46 tokens_per_ep_rank=[32776, 37, 33606, 32750, 35, 65106, 32768, 65066] avg=32768.0000 max=65106 max_rank=5 min=35 min_rank=4 imbalance_ratio=1.986877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=48 tokens_per_ep_rank=[96544, 65500, 18, 32643, 2597, 59046, 2482, 3314] avg=32768.0000 max=96544 max_rank=0 min=18 min_rank=2 imbalance_ratio=2.946289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] avg=32768.0000 max=33096 max_rank=0 min=31254 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63448 max_load_reduction_pct=0.657193 moved_to_replicas=121802 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=46 tokens_per_ep_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32010 max_load_reduction_pct=0.491660 moved_to_replicas=64490 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=46 tokens_per_ep_rank=[32776, 32517, 33096, 32750, 32045, 33096, 32768, 33096] avg=32768.0000 max=33096 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32010 max_load_reduction_pct=0.491660 moved_to_replicas=64490 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] avg=32768.0000 max=33096 max_rank=0 min=31254 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63448 max_load_reduction_pct=0.657193 moved_to_replicas=121802 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=47 tokens_per_ep_rank=[106, 1238, 32768, 68129, 32751, 32761, 32623, 61768] avg=32768.0000 max=68129 max_rank=3 min=106 min_rank=0 imbalance_ratio=2.079132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=47 tokens_per_ep_rank=[106, 1238, 32768, 68129, 32751, 32761, 32623, 61768] avg=32768.0000 max=68129 max_rank=3 min=106 min_rank=0 imbalance_ratio=2.079132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=47 tokens_per_ep_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] avg=32768.0000 max=33096 max_rank=3 min=32195 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35033 max_load_reduction_pct=0.514216 moved_to_replicas=63705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=47 tokens_per_ep_rank=[32854, 32195, 32768, 33096, 32751, 32761, 32623, 33096] avg=32768.0000 max=33096 max_rank=3 min=32195 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35033 max_load_reduction_pct=0.514216 moved_to_replicas=63705 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=48 tokens_per_ep_rank=[96544, 65500, 18, 32643, 2597, 59046, 2482, 3314] avg=32768.0000 max=96544 max_rank=0 min=18 min_rank=2 imbalance_ratio=2.946289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=48 tokens_per_ep_rank=[96544, 65500, 18, 32643, 2597, 59046, 2482, 3314] avg=32768.0000 max=96544 max_rank=0 min=18 min_rank=2 imbalance_ratio=2.946289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] avg=32768.0000 max=33096 max_rank=0 min=31254 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63448 max_load_reduction_pct=0.657193 moved_to_replicas=121802 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=48 tokens_per_ep_rank=[33096, 33096, 32767, 32643, 33096, 33096, 33096, 31254] avg=32768.0000 max=33096 max_rank=0 min=31254 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63448 max_load_reduction_pct=0.657193 moved_to_replicas=121802 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=48 total_expected_recv=1570956 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378112 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=48 total_expected_recv=1574537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374528 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=48 total_expected_recv=1565830 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380928 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=48 total_expected_recv=1568660 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=380928 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=48 total_expected_recv=1574325 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373120 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=48 total_expected_recv=1576158 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372480 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=48 total_expected_recv=1574643 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374528 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=48 total_expected_recv=1577803 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=369152 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:47:06] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 744.0 | throughput per GPU (TFLOP/s/GPU): 119.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.798977E+00 | loss scale: 1.0 | grad norm: 3.168 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34432 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32206 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=31142 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=30473 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=30472 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33027 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=31142 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33027 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=31119 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=31119 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=31347 expected_overflow=0 exact_recv_per_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=31347 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32222 expected_overflow=0 exact_recv_per_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32222 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=30928 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=30928 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=30775 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=30775 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=30778 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=31980 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=31980 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32119 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32119 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32398 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32398 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=31780 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=31780 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=30778 actual_valid_recv=31488 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=30769 expected_overflow=0 exact_recv_per_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=31762 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32184 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=31762 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32184 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32071 expected_overflow=0 exact_recv_per_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=30769 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32071 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=31498 expected_overflow=0 exact_recv_per_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=31498 actual_valid_recv=32128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33077 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33077 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=31083 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=31083 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=31701 expected_overflow=0 exact_recv_per_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=31701 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=30806 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32998 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=30806 actual_valid_recv=31104 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=31104 slack_slots=9856 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32998 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32399 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32399 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=31503 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=31503 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32286 expected_overflow=0 exact_recv_per_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32286 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=31853 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=31853 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32400 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32400 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=31443 expected_overflow=0 exact_recv_per_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=31443 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=40960 expected_recv=31774 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=40960 expected_recv=31774 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32190 expected_overflow=0 exact_recv_per_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32190 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=40960 expected_recv=31117 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=40960 expected_recv=31117 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=40960 expected_recv=31881 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=40960 expected_recv=31881 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32384 expected_overflow=0 exact_recv_per_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32384 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=40960 expected_recv=31710 expected_overflow=0 exact_recv_per_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=40960 expected_recv=31710 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=40960 expected_recv=30992 expected_overflow=0 exact_recv_per_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32137 expected_overflow=0 exact_recv_per_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32137 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32275 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32275 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=40960 expected_recv=30992 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32323 expected_overflow=0 exact_recv_per_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=40960 expected_recv=31272 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32323 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=40960 expected_recv=31272 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=40960 expected_recv=31116 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=40960 expected_recv=31116 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32349 expected_overflow=0 exact_recv_per_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32349 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32382 expected_overflow=0 exact_recv_per_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32382 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=40960 expected_recv=31518 expected_overflow=0 exact_recv_per_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=40960 expected_recv=31518 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32042 expected_overflow=0 exact_recv_per_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32042 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32514 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=40960 expected_recv=31570 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=40960 expected_recv=31570 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=40960 expected_recv=31341 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=40960 expected_recv=31341 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32027 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=40960 expected_recv=31446 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32027 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=40960 expected_recv=31446 actual_valid_recv=31744 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32007 expected_overflow=0 exact_recv_per_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33082 expected_overflow=0 exact_recv_per_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32349 expected_overflow=0 exact_recv_per_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32259 expected_overflow=0 exact_recv_per_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32461 expected_overflow=0 exact_recv_per_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32007 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33082 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32349 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32259 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32461 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32188 expected_overflow=0 exact_recv_per_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=40960 expected_recv=31791 expected_overflow=0 exact_recv_per_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32304 expected_overflow=0 exact_recv_per_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32188 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=40960 expected_recv=31791 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32304 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=40960 expected_recv=31589 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=40960 expected_recv=31589 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=40960 expected_recv=31693 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=40960 expected_recv=31693 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=40960 expected_recv=31962 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=40960 expected_recv=31962 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32240 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32240 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32407 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32001 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32407 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32001 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32110 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=40960 expected_recv=31853 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=40960 expected_recv=31853 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=40960 expected_recv=31789 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=40960 expected_recv=31789 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32318 expected_overflow=0 exact_recv_per_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33074 expected_overflow=0 exact_recv_per_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32318 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33074 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=23 calls=1 budget=40960 expected_recv=31476 expected_overflow=0 exact_recv_per_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=23 calls=1 budget=40960 expected_recv=31476 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32110 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32084 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32365 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32365 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32142 expected_overflow=0 exact_recv_per_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32142 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=26 calls=1 budget=40960 expected_recv=31246 expected_overflow=0 exact_recv_per_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=26 calls=1 budget=40960 expected_recv=31246 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=25 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=25 calls=1 budget=40960 expected_recv=31331 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=25 calls=1 budget=40960 expected_recv=31331 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33064 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33064 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=26 calls=1 budget=40960 expected_recv=30830 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32223 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=26 calls=1 budget=40960 expected_recv=30830 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32223 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=31232 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=28 calls=1 budget=40960 expected_recv=31972 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=27 calls=1 budget=40960 expected_recv=31958 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=28 calls=1 budget=40960 expected_recv=31972 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=27 calls=1 budget=40960 expected_recv=31958 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=28 calls=1 budget=40960 expected_recv=31526 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=27 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=28 calls=1 budget=40960 expected_recv=31526 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32497 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32497 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=30 calls=1 budget=40960 expected_recv=31221 expected_overflow=0 exact_recv_per_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=30 calls=1 budget=40960 expected_recv=31221 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32267 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32267 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=30 calls=1 budget=40960 expected_recv=31461 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=30 calls=1 budget=40960 expected_recv=31461 actual_valid_recv=31616 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31616 slack_slots=9344 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=29 calls=1 budget=40960 expected_recv=31853 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=29 calls=1 budget=40960 expected_recv=31853 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32197 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32197 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32489 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32489 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=31744 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33008 expected_overflow=0 exact_recv_per_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=29 calls=1 budget=40960 expected_recv=31832 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33008 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=29 calls=1 budget=40960 expected_recv=31832 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32360 expected_overflow=0 exact_recv_per_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32360 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32373 expected_overflow=0 exact_recv_per_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32516 expected_overflow=0 exact_recv_per_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32373 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32516 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32983 expected_overflow=0 exact_recv_per_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32172 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32172 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32082 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32082 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32983 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=32 calls=1 budget=40960 expected_recv=31891 expected_overflow=0 exact_recv_per_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=32 calls=1 budget=40960 expected_recv=31891 actual_valid_recv=32128 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32387 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32387 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32103 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32103 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=33 calls=1 budget=40960 expected_recv=31874 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=33 calls=1 budget=40960 expected_recv=31874 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32158 expected_overflow=0 exact_recv_per_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32158 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32449 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32123 expected_overflow=0 exact_recv_per_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32123 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32357 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32357 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32134 expected_overflow=0 exact_recv_per_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32134 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32449 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32442 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32411 expected_overflow=0 exact_recv_per_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32411 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32057 expected_overflow=0 exact_recv_per_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32057 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32112 expected_overflow=0 exact_recv_per_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32196 expected_overflow=0 exact_recv_per_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32112 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32196 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32423 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32423 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32141 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32141 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=40 calls=1 budget=40960 expected_recv=31721 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=40 calls=1 budget=40960 expected_recv=31721 actual_valid_recv=31872 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=40 calls=1 budget=40960 expected_recv=31137 expected_overflow=0 exact_recv_per_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31872 slack_slots=9088 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=41 calls=1 budget=40960 expected_recv=31894 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=41 calls=1 budget=40960 expected_recv=31894 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32974 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=40 calls=1 budget=40960 expected_recv=31137 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=31488 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32382 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32382 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=42 calls=1 budget=40960 expected_recv=31716 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=43 calls=1 budget=40960 expected_recv=31787 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32974 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=42 calls=1 budget=40960 expected_recv=31716 actual_valid_recv=32000 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=43 calls=1 budget=40960 expected_recv=31787 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32455 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32174 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=41 calls=1 budget=40960 expected_recv=31088 expected_overflow=0 exact_recv_per_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=41 calls=1 budget=40960 expected_recv=31088 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=31360 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33013 expected_overflow=0 exact_recv_per_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33013 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=43 calls=1 budget=40960 expected_recv=31648 expected_overflow=0 exact_recv_per_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32111 expected_overflow=0 exact_recv_per_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32111 actual_valid_recv=32256 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32115 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32115 actual_valid_recv=32384 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32384 slack_slots=8576 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=43 calls=1 budget=40960 expected_recv=31648 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=42 calls=1 budget=40960 expected_recv=31753 expected_overflow=0 exact_recv_per_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=45 calls=1 budget=40960 expected_recv=31454 expected_overflow=0 exact_recv_per_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=42 calls=1 budget=40960 expected_recv=31753 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=45 calls=1 budget=40960 expected_recv=31454 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32000 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=46 calls=1 budget=40960 expected_recv=31770 expected_overflow=0 exact_recv_per_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=46 calls=1 budget=40960 expected_recv=31770 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32256 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=32640 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=46 calls=1 budget=40960 expected_recv=31752 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=46 calls=1 budget=40960 expected_recv=31752 actual_valid_recv=32128 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[23229, 28863, 33108, 45049, 42188, 25002, 25689, 39016] avg=32768.0000 max=45049 max_rank=3 min=23229 min_rank=0 imbalance_ratio=1.374786 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32398 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32398 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[23524, 25855, 37049, 40661, 42496, 26042, 29841, 36676] avg=32768.0000 max=42496 max_rank=4 min=23524 min_rank=0 imbalance_ratio=1.296875 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32346, 32112, 33096, 33096, 33096, 32206, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=32112 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=9399 max_load_reduction_pct=0.221174 moved_to_replicas=24498 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[39914, 1535, 16771, 35143, 42510, 83058, 30449, 12764] avg=32768.0000 max=83058 max_rank=5 min=1535 min_rank=1 imbalance_ratio=2.534729 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31142, 33096, 33096, 33095, 32496, 33027] avg=32768.0000 max=33096 max_rank=0 min=31142 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49962 max_load_reduction_pct=0.601531 moved_to_replicas=68241 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[69794, 13460, 41236, 1778, 35814, 56631, 14869, 28562] avg=32768.0000 max=69794 max_rank=0 min=1778 min_rank=3 imbalance_ratio=2.129944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 31119, 33096, 33096, 33096, 32449] avg=32768.0000 max=33096 max_rank=0 min=31119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36698 max_load_reduction_pct=0.525805 moved_to_replicas=71091 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[60692, 29298, 39771, 27775, 27726, 23815, 2167, 50900] avg=32768.0000 max=60692 max_rank=0 min=2167 min_rank=6 imbalance_ratio=1.852173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 30928, 33096, 33096, 32639, 33096] avg=32768.0000 max=33097 max_rank=0 min=30928 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27595 max_load_reduction_pct=0.454673 moved_to_replicas=52075 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[44519, 8511, 58230, 100367, 45787, 157, 36, 4537] avg=32768.0000 max=100367 max_rank=3 min=36 min_rank=6 imbalance_ratio=3.062958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 33095, 30775, 32793, 33097] avg=32768.0000 max=33097 max_rank=1 min=30775 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=67270 max_load_reduction_pct=0.670240 moved_to_replicas=116519 active_clone_links=8 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[74945, 10068, 30778, 48877, 36490, 57148, 3730, 108] avg=32768.0000 max=74945 max_rank=0 min=108 min_rank=7 imbalance_ratio=2.287140 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[33098, 33096, 30778, 33096, 33096, 33096, 33023, 32861] avg=32768.0000 max=33098 max_rank=0 min=30778 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=41847 max_load_reduction_pct=0.558369 moved_to_replicas=85076 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[36117, 26859, 21553, 66589, 44, 19613, 32184, 59185] avg=32768.0000 max=66589 max_rank=3 min=44 min_rank=4 imbalance_ratio=2.032135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[33096, 31762, 33096, 33096, 32718, 33096, 32184, 33096] avg=32768.0000 max=33096 max_rank=0 min=31762 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33493 max_load_reduction_pct=0.502981 moved_to_replicas=62603 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[64071, 108, 35778, 3213, 32772, 43188, 30146, 52868] avg=32768.0000 max=64071 max_rank=0 min=108 min_rank=1 imbalance_ratio=1.955292 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[33096, 31083, 33096, 33077, 32772, 33096, 32828, 33096] avg=32768.0000 max=33096 max_rank=0 min=31083 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30975 max_load_reduction_pct=0.483448 moved_to_replicas=63521 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[23580, 72425, 1263, 63460, 32998, 19071, 49248, 99] avg=32768.0000 max=72425 max_rank=1 min=99 min_rank=7 imbalance_ratio=2.210236 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 32998, 30806, 33096, 32860] avg=32768.0000 max=33096 max_rank=0 min=30806 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39329 max_load_reduction_pct=0.543031 moved_to_replicas=85845 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[1382, 45605, 60936, 48992, 29660, 26304, 32763, 16502] avg=32768.0000 max=60936 max_rank=2 min=1382 min_rank=0 imbalance_ratio=1.859619 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 31503, 33096, 32763, 32399] avg=32768.0000 max=33096 max_rank=0 min=31503 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=27840 max_load_reduction_pct=0.456873 moved_to_replicas=56245 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33854, 61915, 35675, 63666, 30943, 3371, 20] avg=32768.0000 max=63666 max_rank=4 min=20 min_rank=7 imbalance_ratio=1.942932 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32700, 33096, 33096, 33096, 33096, 31774, 32190, 33096] avg=32768.0000 max=33096 max_rank=1 min=31774 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30570 max_load_reduction_pct=0.480162 moved_to_replicas=62726 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32646, 57435, 43368, 40017, 878, 37513, 41892, 8395] avg=32768.0000 max=57435 max_rank=1 min=878 min_rank=4 imbalance_ratio=1.752777 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32646, 33097, 33096, 33096, 32137, 33096, 33095, 31881] avg=32768.0000 max=33097 max_rank=1 min=31881 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=24338 max_load_reduction_pct=0.423749 moved_to_replicas=54745 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 39940, 32275, 21, 55757, 12, 68504] avg=32768.0000 max=68504 max_rank=7 min=12 min_rank=6 imbalance_ratio=2.090576 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32726, 32909, 33097, 32275, 32623, 33095, 32323, 33096] avg=32768.0000 max=33097 max_rank=2 min=32275 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=35407 max_load_reduction_pct=0.516860 moved_to_replicas=64913 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[21326, 93323, 50624, 31059, 43, 172, 85, 65512] avg=32768.0000 max=93323 max_rank=1 min=43 min_rank=4 imbalance_ratio=2.847992 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[33097, 33097, 33095, 31272, 32804, 32588, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31272 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60226 max_load_reduction_pct=0.645350 moved_to_replicas=110171 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 10695, 7, 70545, 65568, 51241, 5] avg=32768.0000 max=70545 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.152863 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32967, 31116, 32919, 33097, 33095, 33096, 33096, 32758] avg=32768.0000 max=33097 max_rank=3 min=31116 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37448 max_load_reduction_pct=0.530838 moved_to_replicas=88066 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[41476, 25570, 65509, 120, 30716, 65766, 32772, 215] avg=32768.0000 max=65766 max_rank=5 min=120 min_rank=3 imbalance_ratio=2.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[33095, 33097, 33096, 32790, 31570, 33096, 32772, 32628] avg=32768.0000 max=33097 max_rank=1 min=31570 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32669 max_load_reduction_pct=0.496746 moved_to_replicas=73463 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[6, 11374, 95, 95345, 15834, 93292, 32556, 13642] avg=32768.0000 max=95345 max_rank=3 min=6 min_rank=0 imbalance_ratio=2.909698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[32767, 33095, 31341, 33096, 33096, 33097, 32556, 33096] avg=32768.0000 max=33097 max_rank=5 min=31341 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62248 max_load_reduction_pct=0.652871 moved_to_replicas=122445 active_clone_links=7 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[33151, 32709, 16, 34062, 63818, 65438, 31972, 978] avg=32768.0000 max=65438 max_rank=5 min=16 min_rank=2 imbalance_ratio=1.997009 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[33096, 32709, 32358, 33096, 33096, 33096, 32027, 32666] avg=32768.0000 max=33096 max_rank=0 min=32027 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=64085 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[33636, 0, 32782, 40428, 32773, 21836, 15569, 85120] avg=32768.0000 max=85120 max_rank=7 min=0 min_rank=1 imbalance_ratio=2.597656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[33096, 32759, 32782, 33096, 32773, 31446, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31446 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=52023 max_load_reduction_pct=0.611172 moved_to_replicas=59896 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[32779, 63564, 24316, 36764, 13, 32485, 32908, 39315] avg=32768.0000 max=63564 max_rank=1 min=13 min_rank=4 imbalance_ratio=1.939819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[32779, 33096, 31589, 33096, 33096, 32485, 32908, 33095] avg=32768.0000 max=33096 max_rank=1 min=31589 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=30468 max_load_reduction_pct=0.479328 moved_to_replicas=40355 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[30505, 68900, 12, 27260, 65014, 32835, 4908, 32710] avg=32768.0000 max=68900 max_rank=1 min=12 min_rank=2 imbalance_ratio=2.102661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[31693, 33096, 32522, 33097, 33096, 32835, 33095, 32710] avg=32768.0000 max=33097 max_rank=3 min=31693 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35803 max_load_reduction_pct=0.519637 moved_to_replicas=67722 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[98252, 12, 28448, 7, 65574, 33480, 3748, 32623] avg=32768.0000 max=98252 max_rank=0 min=7 min_rank=3 imbalance_ratio=2.998413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[33096, 32407, 31962, 32768, 33096, 33096, 33096, 32623] avg=32768.0000 max=33096 max_rank=0 min=31962 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65156 max_load_reduction_pct=0.663152 moved_to_replicas=98018 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[59116, 65530, 65534, 5981, 58540, 3, 644, 6796] avg=32768.0000 max=65534 max_rank=2 min=3 min_rank=5 imbalance_ratio=1.999939 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[33096, 33096, 33096, 32001, 33096, 32441, 33078, 32240] avg=32768.0000 max=33096 max_rank=0 min=32001 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=116336 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[28033, 32638, 32110, 47136, 1808, 54891, 65506, 22] avg=32768.0000 max=65506 max_rank=6 min=22 min_rank=7 imbalance_ratio=1.999084 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[32579, 32638, 32110, 33095, 33098, 33096, 33096, 32432] avg=32768.0000 max=33098 max_rank=4 min=32110 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32408 max_load_reduction_pct=0.494733 moved_to_replicas=68245 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=25 tokens_per_ep_rank=[428, 65455, 65328, 32836, 32568, 6, 32749, 32774] avg=32768.0000 max=65455 max_rank=1 min=6 min_rank=5 imbalance_ratio=1.997528 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=25 tokens_per_ep_rank=[32660, 33096, 33096, 32836, 32568, 32365, 32749, 32774] avg=32768.0000 max=33096 max_rank=1 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32359 max_load_reduction_pct=0.494370 moved_to_replicas=64591 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=26 tokens_per_ep_rank=[41, 32771, 32722, 32764, 65549, 17, 65519, 32761] avg=32768.0000 max=65549 max_rank=4 min=17 min_rank=5 imbalance_ratio=2.000397 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=26 tokens_per_ep_rank=[32464, 32771, 32722, 32764, 33096, 32470, 33096, 32761] avg=32768.0000 max=33096 max_rank=4 min=32464 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32453 max_load_reduction_pct=0.495095 moved_to_replicas=64876 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=27 tokens_per_ep_rank=[49, 60408, 65699, 16, 79607, 6, 80, 56279] avg=32768.0000 max=79607 max_rank=4 min=6 min_rank=5 imbalance_ratio=2.429413 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=27 tokens_per_ep_rank=[32652, 33098, 33096, 33097, 33096, 32764, 31246, 33095] avg=32768.0000 max=33098 max_rank=1 min=31246 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=46509 max_load_reduction_pct=0.584233 moved_to_replicas=129609 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=28 tokens_per_ep_rank=[0, 31958, 65, 120912, 65785, 32709, 10703, 12] avg=32768.0000 max=120912 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.689941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=28 tokens_per_ep_rank=[32761, 31958, 32657, 33095, 33096, 32709, 33095, 32773] avg=32768.0000 max=33096 max_rank=4 min=31958 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=87816 max_load_reduction_pct=0.726280 moved_to_replicas=120505 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=29 tokens_per_ep_rank=[44926, 99, 65530, 98024, 47539, 5698, 12, 316] avg=32768.0000 max=98024 max_rank=3 min=12 min_rank=6 imbalance_ratio=2.991455 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=29 tokens_per_ep_rank=[33095, 32267, 33096, 33096, 33096, 31972, 32772, 32750] avg=32768.0000 max=33096 max_rank=2 min=31972 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64928 max_load_reduction_pct=0.662368 moved_to_replicas=123635 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=30 tokens_per_ep_rank=[65532, 66455, 0, 32768, 32752, 6, 31853, 32778] avg=32768.0000 max=66455 max_rank=1 min=0 min_rank=2 imbalance_ratio=2.028046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32761, 32768, 32752, 33040, 31853, 32778] avg=32768.0000 max=33096 max_rank=0 min=31853 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33359 max_load_reduction_pct=0.501979 moved_to_replicas=65795 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=31 tokens_per_ep_rank=[18, 23, 66390, 6, 65635, 32762, 0, 97310] avg=32768.0000 max=97310 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.969666 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=31 tokens_per_ep_rank=[32779, 33095, 33096, 31461, 33096, 32762, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=31461 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64214 max_load_reduction_pct=0.659891 moved_to_replicas=130047 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=32 tokens_per_ep_rank=[8, 32768, 35715, 33457, 32769, 64924, 29741, 32762] avg=32768.0000 max=64924 max_rank=5 min=8 min_rank=0 imbalance_ratio=1.981323 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=32 tokens_per_ep_rank=[32197, 32768, 33096, 33096, 32769, 33096, 32360, 32762] avg=32768.0000 max=33096 max_rank=2 min=32197 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31828 max_load_reduction_pct=0.490235 moved_to_replicas=34808 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=33 tokens_per_ep_rank=[4, 32734, 65574, 6, 32786, 32756, 98273, 11] avg=32768.0000 max=98273 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.999054 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=33 tokens_per_ep_rank=[32765, 32734, 33096, 32422, 32786, 32756, 33096, 32489] avg=32768.0000 max=33096 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65177 max_load_reduction_pct=0.663224 moved_to_replicas=97655 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=34 tokens_per_ep_rank=[65471, 65092, 66027, 6, 12, 8, 0, 65528] avg=32768.0000 max=66027 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.014984 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 33096, 32172, 32387, 32440, 32761, 33096] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32931 max_load_reduction_pct=0.498751 moved_to_replicas=129734 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=35 tokens_per_ep_rank=[18, 29599, 32780, 104190, 0, 32763, 32787, 30007] avg=32768.0000 max=104190 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.179626 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=35 tokens_per_ep_rank=[32779, 33096, 32780, 33096, 32761, 32763, 32787, 32082] avg=32768.0000 max=33096 max_rank=1 min=32082 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=71094 max_load_reduction_pct=0.682350 moved_to_replicas=71094 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=36 tokens_per_ep_rank=[65536, 32762, 10510, 10, 6, 65534, 65530, 22256] avg=32768.0000 max=65536 max_rank=0 min=6 min_rank=4 imbalance_ratio=2.000000 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32762, 33097, 32448, 32446, 33096, 33096, 32103] avg=32768.0000 max=33097 max_rank=2 min=32103 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32439 max_load_reduction_pct=0.494980 moved_to_replicas=97312 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=37 tokens_per_ep_rank=[3, 32774, 32777, 629, 64906, 0, 98282, 32773] avg=32768.0000 max=98282 max_rank=6 min=0 min_rank=5 imbalance_ratio=2.999329 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=37 tokens_per_ep_rank=[32428, 32774, 32777, 32439, 33096, 32761, 33096, 32773] avg=32768.0000 max=33096 max_rank=4 min=32428 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65186 max_load_reduction_pct=0.663255 moved_to_replicas=96996 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=38 tokens_per_ep_rank=[32769, 6106, 98293, 32766, 32456, 13, 6, 59735] avg=32768.0000 max=98293 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.999664 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=38 tokens_per_ep_rank=[32769, 32747, 33096, 32766, 32456, 32449, 32767, 33094] avg=32768.0000 max=33096 max_rank=2 min=32449 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65197 max_load_reduction_pct=0.663292 moved_to_replicas=91836 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=39 tokens_per_ep_rank=[24, 32766, 98284, 32774, 8, 6106, 92174, 8] avg=32768.0000 max=98284 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.999390 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=39 tokens_per_ep_rank=[32785, 32766, 33096, 32774, 32769, 32423, 33096, 32435] avg=32768.0000 max=33096 max_rank=2 min=32423 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=124266 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=40 tokens_per_ep_rank=[32141, 66314, 32758, 64377, 79, 33705, 32770, 0] avg=32768.0000 max=66314 max_rank=1 min=0 min_rank=7 imbalance_ratio=2.023743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=40 tokens_per_ep_rank=[32141, 33096, 32758, 33096, 32426, 33096, 32770, 32761] avg=32768.0000 max=33096 max_rank=1 min=32141 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33218 max_load_reduction_pct=0.500920 moved_to_replicas=65108 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=41 tokens_per_ep_rank=[32793, 6, 47582, 30451, 3, 32813, 52970, 65526] avg=32768.0000 max=65526 max_rank=7 min=3 min_rank=4 imbalance_ratio=1.999695 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=41 tokens_per_ep_rank=[32793, 33094, 33098, 31721, 32433, 32813, 33096, 33096] avg=32768.0000 max=33098 max_rank=2 min=31721 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32428 max_load_reduction_pct=0.494888 moved_to_replicas=66790 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=42 tokens_per_ep_rank=[27038, 37951, 17106, 65545, 48964, 32766, 6, 32768] avg=32768.0000 max=65545 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000275 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=42 tokens_per_ep_rank=[31894, 33095, 32974, 33096, 33096, 32766, 32455, 32768] avg=32768.0000 max=33096 max_rank=3 min=31894 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32449 max_load_reduction_pct=0.495064 moved_to_replicas=53172 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=43 tokens_per_ep_rank=[81966, 20259, 48, 12, 32774, 3, 65222, 61860] avg=32768.0000 max=81966 max_rank=0 min=3 min_rank=5 imbalance_ratio=2.501404 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=43 tokens_per_ep_rank=[33095, 33098, 32174, 33095, 32774, 31716, 33096, 33096] avg=32768.0000 max=33098 max_rank=1 min=31716 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=48868 max_load_reduction_pct=0.596198 moved_to_replicas=109760 active_clone_links=6 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=44 tokens_per_ep_rank=[32755, 14, 91024, 32766, 390, 32771, 72404, 20] avg=32768.0000 max=91024 max_rank=2 min=14 min_rank=1 imbalance_ratio=2.777832 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=44 tokens_per_ep_rank=[32755, 32775, 33097, 32766, 33096, 32771, 33097, 31787] avg=32768.0000 max=33097 max_rank=2 min=31787 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57927 max_load_reduction_pct=0.636393 moved_to_replicas=97236 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 61938, 0, 36365] avg=32768.0000 max=61938 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.890198 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=45 tokens_per_ep_rank=[32774, 32763, 32768, 32762, 32774, 33096, 32111, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28842 max_load_reduction_pct=0.465659 moved_to_replicas=32111 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=46 tokens_per_ep_rank=[32762, 14, 12, 68924, 32768, 32771, 94893, 0] avg=32768.0000 max=94893 max_rank=6 min=0 min_rank=7 imbalance_ratio=2.895905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=46 tokens_per_ep_rank=[32762, 32775, 32115, 33096, 32768, 32771, 33096, 32761] avg=32768.0000 max=33096 max_rank=3 min=32115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61797 max_load_reduction_pct=0.651228 moved_to_replicas=97625 active_clone_links=4 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=47 tokens_per_ep_rank=[17, 13299, 32768, 98634, 32768, 32790, 29077, 22791] avg=32768.0000 max=98634 max_rank=3 min=17 min_rank=0 imbalance_ratio=3.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=47 tokens_per_ep_rank=[32778, 33096, 32768, 33096, 32768, 32790, 31752, 33096] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65538 max_load_reduction_pct=0.664456 moved_to_replicas=65538 active_clone_links=5 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=48 tokens_per_ep_rank=[3853, 32398, 46340, 21, 32693, 81269, 32755, 32815] avg=32768.0000 max=81269 max_rank=5 min=21 min_rank=3 imbalance_ratio=2.480133 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=48 tokens_per_ep_rank=[32509, 32398, 33096, 32782, 32693, 33096, 32755, 32815] avg=32768.0000 max=33096 max_rank=2 min=32398 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48173 max_load_reduction_pct=0.592760 moved_to_replicas=61417 active_clone_links=3 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=48 total_expected_recv=1573727 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373248 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=48 total_expected_recv=1574569 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374016 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=48 total_expected_recv=1571606 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376064 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=48 total_expected_recv=1568576 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379392 peak_demand=33097 required_capacity=1.010040 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=48 total_expected_recv=1572352 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375936 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=48 total_expected_recv=1575436 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373120 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=48 total_expected_recv=1574252 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372864 peak_demand=33098 required_capacity=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=48 total_expected_recv=1572394 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377088 peak_demand=33097 required_capacity=1.010040 - [2026-08-26 15:01:28] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 722.7 | throughput per GPU (TFLOP/s/GPU): 123.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.744781E+00 | loss scale: 1.0 | grad norm: 2.735 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | -[after training is done] datetime: 2026-08-26 15:01:28 -W0826 15:01:28.747000 49721 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 15:01:28.748000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49814 closing signal SIGTERM -W0826 15:01:28.748000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49815 closing signal SIGTERM -W0826 15:01:28.749000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49816 closing signal SIGTERM -W0826 15:01:28.749000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49817 closing signal SIGTERM -W0826 15:01:28.751000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49818 closing signal SIGTERM -W0826 15:01:28.752000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49819 closing signal SIGTERM -W0826 15:01:28.753000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49820 closing signal SIGTERM -W0826 15:01:28.753000 49721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 49821 closing signal SIGTERM +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=47 calls=1 budget=40960 expected_recv=31371 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[23229, 28863, 33108, 45049, 42188, 25002, 25689, 39016] avg=32768.0000 max=45049 max_rank=3 min=23229 min_rank=0 imbalance_ratio=1.374786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=11952 max_load_reduction_pct=0.265311 moved_to_replicas=26977 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=47 calls=1 budget=40960 expected_recv=31371 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=32128 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[23229, 28863, 33108, 45049, 42188, 25002, 25689, 39016] avg=32768.0000 max=45049 max_rank=3 min=23229 min_rank=0 imbalance_ratio=1.374786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=11952 max_load_reduction_pct=0.265311 moved_to_replicas=26977 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[23229, 28863, 33108, 45049, 42188, 25002, 25689, 39016] avg=32768.0000 max=45049 max_rank=3 min=23229 min_rank=0 imbalance_ratio=1.374786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[23229, 28863, 33108, 45049, 42188, 25002, 25689, 39016] avg=32768.0000 max=45049 max_rank=3 min=23229 min_rank=0 imbalance_ratio=1.374786 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[44157, 696, 22185, 33552, 33729, 63870, 44282, 19673] avg=32768.0000 max=63870 max_rank=5 min=696 min_rank=1 imbalance_ratio=1.949158 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=46 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=11952 max_load_reduction_pct=0.265311 moved_to_replicas=26977 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[23229, 28863, 33108, 45049, 42188, 25002, 25689, 39016] avg=32768.0000 max=45049 max_rank=3 min=23229 min_rank=0 imbalance_ratio=1.374786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=11952 max_load_reduction_pct=0.265311 moved_to_replicas=26977 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[44157, 696, 22185, 33552, 33729, 63870, 44282, 19673] avg=32768.0000 max=63870 max_rank=5 min=696 min_rank=1 imbalance_ratio=1.949158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[23229, 28863, 33108, 45049, 42188, 25002, 25689, 39016] avg=32768.0000 max=45049 max_rank=3 min=23229 min_rank=0 imbalance_ratio=1.374786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30772 max_load_reduction_pct=0.481791 moved_to_replicas=54110 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=11952 max_load_reduction_pct=0.265311 moved_to_replicas=26977 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=11952 max_load_reduction_pct=0.265311 moved_to_replicas=26977 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30772 max_load_reduction_pct=0.481791 moved_to_replicas=54110 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[44157, 696, 22185, 33552, 33729, 63870, 44282, 19673] avg=32768.0000 max=63870 max_rank=5 min=696 min_rank=1 imbalance_ratio=1.949158 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=11952 max_load_reduction_pct=0.265311 moved_to_replicas=26977 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[44157, 696, 22185, 33552, 33729, 63870, 44282, 19673] avg=32768.0000 max=63870 max_rank=5 min=696 min_rank=1 imbalance_ratio=1.949158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[83397, 29601, 13914, 4437, 26361, 34502, 35621, 34311] avg=32768.0000 max=83397 max_rank=0 min=4437 min_rank=3 imbalance_ratio=2.545074 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[44157, 696, 22185, 33552, 33729, 63870, 44282, 19673] avg=32768.0000 max=63870 max_rank=5 min=696 min_rank=1 imbalance_ratio=1.949158 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30772 max_load_reduction_pct=0.481791 moved_to_replicas=54110 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[44157, 696, 22185, 33552, 33729, 63870, 44282, 19673] avg=32768.0000 max=63870 max_rank=5 min=696 min_rank=1 imbalance_ratio=1.949158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30772 max_load_reduction_pct=0.481791 moved_to_replicas=54110 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31347 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=50301 max_load_reduction_pct=0.603151 moved_to_replicas=55447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[83397, 29601, 13914, 4437, 26361, 34502, 35621, 34311] avg=32768.0000 max=83397 max_rank=0 min=4437 min_rank=3 imbalance_ratio=2.545074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30772 max_load_reduction_pct=0.481791 moved_to_replicas=54110 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[44157, 696, 22185, 33552, 33729, 63870, 44282, 19673] avg=32768.0000 max=63870 max_rank=5 min=696 min_rank=1 imbalance_ratio=1.949158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30772 max_load_reduction_pct=0.481791 moved_to_replicas=54110 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[83397, 29601, 13914, 4437, 26361, 34502, 35621, 34311] avg=32768.0000 max=83397 max_rank=0 min=4437 min_rank=3 imbalance_ratio=2.545074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31347 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=50301 max_load_reduction_pct=0.603151 moved_to_replicas=55447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[23229, 28863, 33108, 45049, 42188, 25002, 25689, 39016] avg=32768.0000 max=45049 max_rank=3 min=23229 min_rank=0 imbalance_ratio=1.374786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[83397, 29601, 13914, 4437, 26361, 34502, 35621, 34311] avg=32768.0000 max=83397 max_rank=0 min=4437 min_rank=3 imbalance_ratio=2.545074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[35877, 15, 32884, 7781, 3245, 32119, 88190, 62033] avg=32768.0000 max=88190 max_rank=6 min=15 min_rank=1 imbalance_ratio=2.691345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30772 max_load_reduction_pct=0.481791 moved_to_replicas=54110 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[83397, 29601, 13914, 4437, 26361, 34502, 35621, 34311] avg=32768.0000 max=83397 max_rank=0 min=4437 min_rank=3 imbalance_ratio=2.545074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31347 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=50301 max_load_reduction_pct=0.603151 moved_to_replicas=55447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33095, 33096, 33095, 33096, 30473, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30473 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=11952 max_load_reduction_pct=0.265311 moved_to_replicas=26977 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31347 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=50301 max_load_reduction_pct=0.603151 moved_to_replicas=55447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[83397, 29601, 13914, 4437, 26361, 34502, 35621, 34311] avg=32768.0000 max=83397 max_rank=0 min=4437 min_rank=3 imbalance_ratio=2.545074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31980 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55094 max_load_reduction_pct=0.624719 moved_to_replicas=86812 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[35877, 15, 32884, 7781, 3245, 32119, 88190, 62033] avg=32768.0000 max=88190 max_rank=6 min=15 min_rank=1 imbalance_ratio=2.691345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31347 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=50301 max_load_reduction_pct=0.603151 moved_to_replicas=55447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[83397, 29601, 13914, 4437, 26361, 34502, 35621, 34311] avg=32768.0000 max=83397 max_rank=0 min=4437 min_rank=3 imbalance_ratio=2.545074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[35877, 15, 32884, 7781, 3245, 32119, 88190, 62033] avg=32768.0000 max=88190 max_rank=6 min=15 min_rank=1 imbalance_ratio=2.691345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31347 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=50301 max_load_reduction_pct=0.603151 moved_to_replicas=55447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31980 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55094 max_load_reduction_pct=0.624719 moved_to_replicas=86812 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[44157, 696, 22185, 33552, 33729, 63870, 44282, 19673] avg=32768.0000 max=63870 max_rank=5 min=696 min_rank=1 imbalance_ratio=1.949158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31347 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=50301 max_load_reduction_pct=0.603151 moved_to_replicas=55447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[35877, 15, 32884, 7781, 3245, 32119, 88190, 62033] avg=32768.0000 max=88190 max_rank=6 min=15 min_rank=1 imbalance_ratio=2.691345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[38213, 2296, 76553, 34087, 45680, 26663, 9, 38643] avg=32768.0000 max=76553 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.336212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[35877, 15, 32884, 7781, 3245, 32119, 88190, 62033] avg=32768.0000 max=88190 max_rank=6 min=15 min_rank=1 imbalance_ratio=2.691345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31980 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55094 max_load_reduction_pct=0.624719 moved_to_replicas=86812 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=30472 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30772 max_load_reduction_pct=0.481791 moved_to_replicas=54110 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[35877, 15, 32884, 7781, 3245, 32119, 88190, 62033] avg=32768.0000 max=88190 max_rank=6 min=15 min_rank=1 imbalance_ratio=2.691345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31980 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55094 max_load_reduction_pct=0.624719 moved_to_replicas=86812 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=43457 max_load_reduction_pct=0.567672 moved_to_replicas=67696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[38213, 2296, 76553, 34087, 45680, 26663, 9, 38643] avg=32768.0000 max=76553 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.336212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31980 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55094 max_load_reduction_pct=0.624719 moved_to_replicas=86812 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[35877, 15, 32884, 7781, 3245, 32119, 88190, 62033] avg=32768.0000 max=88190 max_rank=6 min=15 min_rank=1 imbalance_ratio=2.691345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[38213, 2296, 76553, 34087, 45680, 26663, 9, 38643] avg=32768.0000 max=76553 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.336212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31980 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55094 max_load_reduction_pct=0.624719 moved_to_replicas=86812 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=43457 max_load_reduction_pct=0.567672 moved_to_replicas=67696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[83397, 29601, 13914, 4437, 26361, 34502, 35621, 34311] avg=32768.0000 max=83397 max_rank=0 min=4437 min_rank=3 imbalance_ratio=2.545074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31980 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55094 max_load_reduction_pct=0.624719 moved_to_replicas=86812 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[73649, 69, 481, 18270, 29915, 156, 46073, 93531] avg=32768.0000 max=93531 max_rank=7 min=69 min_rank=1 imbalance_ratio=2.854340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[38213, 2296, 76553, 34087, 45680, 26663, 9, 38643] avg=32768.0000 max=76553 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.336212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[38213, 2296, 76553, 34087, 45680, 26663, 9, 38643] avg=32768.0000 max=76553 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.336212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=43457 max_load_reduction_pct=0.567672 moved_to_replicas=67696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32222, 33096, 33096, 31347, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31347 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=50301 max_load_reduction_pct=0.603151 moved_to_replicas=55447 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[38213, 2296, 76553, 34087, 45680, 26663, 9, 38643] avg=32768.0000 max=76553 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.336212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=30769 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60434 max_load_reduction_pct=0.646139 moved_to_replicas=113965 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=43457 max_load_reduction_pct=0.567672 moved_to_replicas=67696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[73649, 69, 481, 18270, 29915, 156, 46073, 93531] avg=32768.0000 max=93531 max_rank=7 min=69 min_rank=1 imbalance_ratio=2.854340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=43457 max_load_reduction_pct=0.567672 moved_to_replicas=67696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[38213, 2296, 76553, 34087, 45680, 26663, 9, 38643] avg=32768.0000 max=76553 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.336212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[73649, 69, 481, 18270, 29915, 156, 46073, 93531] avg=32768.0000 max=93531 max_rank=7 min=69 min_rank=1 imbalance_ratio=2.854340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=43457 max_load_reduction_pct=0.567672 moved_to_replicas=67696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=30769 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60434 max_load_reduction_pct=0.646139 moved_to_replicas=113965 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[35877, 15, 32884, 7781, 3245, 32119, 88190, 62033] avg=32768.0000 max=88190 max_rank=6 min=15 min_rank=1 imbalance_ratio=2.691345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[1, 65166, 58218, 24311, 33137, 44913, 303, 36095] avg=32768.0000 max=65166 max_rank=1 min=1 min_rank=0 imbalance_ratio=1.988708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=43457 max_load_reduction_pct=0.567672 moved_to_replicas=67696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[73649, 69, 481, 18270, 29915, 156, 46073, 93531] avg=32768.0000 max=93531 max_rank=7 min=69 min_rank=1 imbalance_ratio=2.854340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[73649, 69, 481, 18270, 29915, 156, 46073, 93531] avg=32768.0000 max=93531 max_rank=7 min=69 min_rank=1 imbalance_ratio=2.854340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=30769 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60434 max_load_reduction_pct=0.646139 moved_to_replicas=113965 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] avg=32768.0000 max=33097 max_rank=5 min=31498 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32069 max_load_reduction_pct=0.492112 moved_to_replicas=72049 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33096, 32777, 32884, 33096, 31980, 32119, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31980 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55094 max_load_reduction_pct=0.624719 moved_to_replicas=86812 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[73649, 69, 481, 18270, 29915, 156, 46073, 93531] avg=32768.0000 max=93531 max_rank=7 min=69 min_rank=1 imbalance_ratio=2.854340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[1, 65166, 58218, 24311, 33137, 44913, 303, 36095] avg=32768.0000 max=65166 max_rank=1 min=1 min_rank=0 imbalance_ratio=1.988708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=30769 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60434 max_load_reduction_pct=0.646139 moved_to_replicas=113965 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=30769 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60434 max_load_reduction_pct=0.646139 moved_to_replicas=113965 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[73649, 69, 481, 18270, 29915, 156, 46073, 93531] avg=32768.0000 max=93531 max_rank=7 min=69 min_rank=1 imbalance_ratio=2.854340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[1, 65166, 58218, 24311, 33137, 44913, 303, 36095] avg=32768.0000 max=65166 max_rank=1 min=1 min_rank=0 imbalance_ratio=1.988708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=30769 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60434 max_load_reduction_pct=0.646139 moved_to_replicas=113965 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] avg=32768.0000 max=33097 max_rank=5 min=31498 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32069 max_load_reduction_pct=0.492112 moved_to_replicas=72049 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[45504, 32677, 62977, 32286, 41736, 46754, 173, 37] avg=32768.0000 max=62977 max_rank=2 min=37 min_rank=7 imbalance_ratio=1.921906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[38213, 2296, 76553, 34087, 45680, 26663, 9, 38643] avg=32768.0000 max=76553 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.336212 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=30769 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60434 max_load_reduction_pct=0.646139 moved_to_replicas=113965 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] avg=32768.0000 max=33097 max_rank=5 min=31498 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32069 max_load_reduction_pct=0.492112 moved_to_replicas=72049 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[1, 65166, 58218, 24311, 33137, 44913, 303, 36095] avg=32768.0000 max=65166 max_rank=1 min=1 min_rank=0 imbalance_ratio=1.988708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[1, 65166, 58218, 24311, 33137, 44913, 303, 36095] avg=32768.0000 max=65166 max_rank=1 min=1 min_rank=0 imbalance_ratio=1.988708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] avg=32768.0000 max=33097 max_rank=4 min=31701 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=29880 max_load_reduction_pct=0.474459 moved_to_replicas=64587 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32486, 33096, 33096, 33096, 31780, 32398, 33096] avg=32768.0000 max=33096 max_rank=0 min=31780 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=43457 max_load_reduction_pct=0.567672 moved_to_replicas=67696 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[1, 65166, 58218, 24311, 33137, 44913, 303, 36095] avg=32768.0000 max=65166 max_rank=1 min=1 min_rank=0 imbalance_ratio=1.988708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] avg=32768.0000 max=33097 max_rank=5 min=31498 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32069 max_load_reduction_pct=0.492112 moved_to_replicas=72049 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[45504, 32677, 62977, 32286, 41736, 46754, 173, 37] avg=32768.0000 max=62977 max_rank=2 min=37 min_rank=7 imbalance_ratio=1.921906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] avg=32768.0000 max=33097 max_rank=5 min=31498 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32069 max_load_reduction_pct=0.492112 moved_to_replicas=72049 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[1, 65166, 58218, 24311, 33137, 44913, 303, 36095] avg=32768.0000 max=65166 max_rank=1 min=1 min_rank=0 imbalance_ratio=1.988708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[45504, 32677, 62977, 32286, 41736, 46754, 173, 37] avg=32768.0000 max=62977 max_rank=2 min=37 min_rank=7 imbalance_ratio=1.921906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] avg=32768.0000 max=33097 max_rank=5 min=31498 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32069 max_load_reduction_pct=0.492112 moved_to_replicas=72049 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[53341, 98245, 20266, 56964, 8, 14, 570, 32736] avg=32768.0000 max=98245 max_rank=1 min=8 min_rank=4 imbalance_ratio=2.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] avg=32768.0000 max=33097 max_rank=4 min=31701 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=29880 max_load_reduction_pct=0.474459 moved_to_replicas=64587 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] avg=32768.0000 max=33097 max_rank=5 min=31498 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32069 max_load_reduction_pct=0.492112 moved_to_replicas=72049 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[73649, 69, 481, 18270, 29915, 156, 46073, 93531] avg=32768.0000 max=93531 max_rank=7 min=69 min_rank=1 imbalance_ratio=2.854340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] avg=32768.0000 max=33097 max_rank=4 min=31701 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=29880 max_load_reduction_pct=0.474459 moved_to_replicas=64587 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[45504, 32677, 62977, 32286, 41736, 46754, 173, 37] avg=32768.0000 max=62977 max_rank=2 min=37 min_rank=7 imbalance_ratio=1.921906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[45504, 32677, 62977, 32286, 41736, 46754, 173, 37] avg=32768.0000 max=62977 max_rank=2 min=37 min_rank=7 imbalance_ratio=1.921906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] avg=32768.0000 max=33097 max_rank=3 min=31853 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65148 max_load_reduction_pct=0.663118 moved_to_replicas=109262 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[45504, 32677, 62977, 32286, 41736, 46754, 173, 37] avg=32768.0000 max=62977 max_rank=2 min=37 min_rank=7 imbalance_ratio=1.921906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[53341, 98245, 20266, 56964, 8, 14, 570, 32736] avg=32768.0000 max=98245 max_rank=1 min=8 min_rank=4 imbalance_ratio=2.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32799, 33097, 30769, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=30769 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60434 max_load_reduction_pct=0.646139 moved_to_replicas=113965 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] avg=32768.0000 max=33097 max_rank=4 min=31701 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=29880 max_load_reduction_pct=0.474459 moved_to_replicas=64587 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] avg=32768.0000 max=33097 max_rank=4 min=31701 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=29880 max_load_reduction_pct=0.474459 moved_to_replicas=64587 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[45504, 32677, 62977, 32286, 41736, 46754, 173, 37] avg=32768.0000 max=62977 max_rank=2 min=37 min_rank=7 imbalance_ratio=1.921906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[53341, 98245, 20266, 56964, 8, 14, 570, 32736] avg=32768.0000 max=98245 max_rank=1 min=8 min_rank=4 imbalance_ratio=2.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[14, 8451, 63649, 32787, 21353, 68407, 32755, 34728] avg=32768.0000 max=68407 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.087616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] avg=32768.0000 max=33097 max_rank=4 min=31701 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=29880 max_load_reduction_pct=0.474459 moved_to_replicas=64587 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] avg=32768.0000 max=33097 max_rank=3 min=31853 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65148 max_load_reduction_pct=0.663118 moved_to_replicas=109262 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] avg=32768.0000 max=33097 max_rank=4 min=31701 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=29880 max_load_reduction_pct=0.474459 moved_to_replicas=64587 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] avg=32768.0000 max=33097 max_rank=3 min=31853 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65148 max_load_reduction_pct=0.663118 moved_to_replicas=109262 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[1, 65166, 58218, 24311, 33137, 44913, 303, 36095] avg=32768.0000 max=65166 max_rank=1 min=1 min_rank=0 imbalance_ratio=1.988708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[53341, 98245, 20266, 56964, 8, 14, 570, 32736] avg=32768.0000 max=98245 max_rank=1 min=8 min_rank=4 imbalance_ratio=2.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[53341, 98245, 20266, 56964, 8, 14, 570, 32736] avg=32768.0000 max=98245 max_rank=1 min=8 min_rank=4 imbalance_ratio=2.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=31443 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35311 max_load_reduction_pct=0.516190 moved_to_replicas=67496 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32071, 33096, 33095, 33095, 33096, 33097, 31498, 33096] avg=32768.0000 max=33097 max_rank=5 min=31498 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32069 max_load_reduction_pct=0.492112 moved_to_replicas=72049 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] avg=32768.0000 max=33097 max_rank=3 min=31853 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65148 max_load_reduction_pct=0.663118 moved_to_replicas=109262 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[14, 8451, 63649, 32787, 21353, 68407, 32755, 34728] avg=32768.0000 max=68407 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.087616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[53341, 98245, 20266, 56964, 8, 14, 570, 32736] avg=32768.0000 max=98245 max_rank=1 min=8 min_rank=4 imbalance_ratio=2.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] avg=32768.0000 max=33097 max_rank=3 min=31853 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65148 max_load_reduction_pct=0.663118 moved_to_replicas=109262 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[53341, 98245, 20266, 56964, 8, 14, 570, 32736] avg=32768.0000 max=98245 max_rank=1 min=8 min_rank=4 imbalance_ratio=2.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[14, 8451, 63649, 32787, 21353, 68407, 32755, 34728] avg=32768.0000 max=68407 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.087616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[8513, 59903, 46877, 85216, 43, 32769, 28808, 15] avg=32768.0000 max=85216 max_rank=3 min=15 min_rank=7 imbalance_ratio=2.600586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=31443 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35311 max_load_reduction_pct=0.516190 moved_to_replicas=67496 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] avg=32768.0000 max=33097 max_rank=3 min=31853 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65148 max_load_reduction_pct=0.663118 moved_to_replicas=109262 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] avg=32768.0000 max=33097 max_rank=3 min=31853 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65148 max_load_reduction_pct=0.663118 moved_to_replicas=109262 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=31443 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35311 max_load_reduction_pct=0.516190 moved_to_replicas=67496 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[14, 8451, 63649, 32787, 21353, 68407, 32755, 34728] avg=32768.0000 max=68407 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.087616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[45504, 32677, 62977, 32286, 41736, 46754, 173, 37] avg=32768.0000 max=62977 max_rank=2 min=37 min_rank=7 imbalance_ratio=1.921906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] avg=32768.0000 max=33097 max_rank=1 min=31117 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=52119 max_load_reduction_pct=0.611610 moved_to_replicas=92708 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[14, 8451, 63649, 32787, 21353, 68407, 32755, 34728] avg=32768.0000 max=68407 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.087616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=31443 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35311 max_load_reduction_pct=0.516190 moved_to_replicas=67496 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32677, 33096, 32286, 33097, 33097, 33094, 31701] avg=32768.0000 max=33097 max_rank=4 min=31701 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=29880 max_load_reduction_pct=0.474459 moved_to_replicas=64587 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[8513, 59903, 46877, 85216, 43, 32769, 28808, 15] avg=32768.0000 max=85216 max_rank=3 min=15 min_rank=7 imbalance_ratio=2.600586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[14, 8451, 63649, 32787, 21353, 68407, 32755, 34728] avg=32768.0000 max=68407 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.087616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[14, 8451, 63649, 32787, 21353, 68407, 32755, 34728] avg=32768.0000 max=68407 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.087616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[8513, 59903, 46877, 85216, 43, 32769, 28808, 15] avg=32768.0000 max=85216 max_rank=3 min=15 min_rank=7 imbalance_ratio=2.600586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=31443 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35311 max_load_reduction_pct=0.516190 moved_to_replicas=67496 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32384, 36014, 464, 2805, 83076, 29983, 77413, 5] avg=32768.0000 max=83076 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.535278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] avg=32768.0000 max=33097 max_rank=1 min=31117 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=52119 max_load_reduction_pct=0.611610 moved_to_replicas=92708 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=31443 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35311 max_load_reduction_pct=0.516190 moved_to_replicas=67496 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=31443 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35311 max_load_reduction_pct=0.516190 moved_to_replicas=67496 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] avg=32768.0000 max=33097 max_rank=1 min=31117 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=52119 max_load_reduction_pct=0.611610 moved_to_replicas=92708 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[8513, 59903, 46877, 85216, 43, 32769, 28808, 15] avg=32768.0000 max=85216 max_rank=3 min=15 min_rank=7 imbalance_ratio=2.600586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[53341, 98245, 20266, 56964, 8, 14, 570, 32736] avg=32768.0000 max=98245 max_rank=1 min=8 min_rank=4 imbalance_ratio=2.998199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] avg=32768.0000 max=33097 max_rank=6 min=31710 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=49979 max_load_reduction_pct=0.601606 moved_to_replicas=97215 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[8513, 59903, 46877, 85216, 43, 32769, 28808, 15] avg=32768.0000 max=85216 max_rank=3 min=15 min_rank=7 imbalance_ratio=2.600586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] avg=32768.0000 max=33097 max_rank=1 min=31117 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=52119 max_load_reduction_pct=0.611610 moved_to_replicas=92708 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32384, 36014, 464, 2805, 83076, 29983, 77413, 5] avg=32768.0000 max=83076 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.535278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[33095, 33096, 31853, 33097, 32771, 32400, 33096, 32736] avg=32768.0000 max=33097 max_rank=3 min=31853 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65148 max_load_reduction_pct=0.663118 moved_to_replicas=109262 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[8513, 59903, 46877, 85216, 43, 32769, 28808, 15] avg=32768.0000 max=85216 max_rank=3 min=15 min_rank=7 imbalance_ratio=2.600586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[8513, 59903, 46877, 85216, 43, 32769, 28808, 15] avg=32768.0000 max=85216 max_rank=3 min=15 min_rank=7 imbalance_ratio=2.600586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32384, 36014, 464, 2805, 83076, 29983, 77413, 5] avg=32768.0000 max=83076 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.535278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] avg=32768.0000 max=33097 max_rank=1 min=31117 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=52119 max_load_reduction_pct=0.611610 moved_to_replicas=92708 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[58971, 13, 926, 32742, 63658, 33513, 22247, 50074] avg=32768.0000 max=63658 max_rank=4 min=13 min_rank=1 imbalance_ratio=1.942688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] avg=32768.0000 max=33097 max_rank=6 min=31710 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=49979 max_load_reduction_pct=0.601606 moved_to_replicas=97215 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] avg=32768.0000 max=33097 max_rank=1 min=31117 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=52119 max_load_reduction_pct=0.611610 moved_to_replicas=92708 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] avg=32768.0000 max=33097 max_rank=1 min=31117 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=52119 max_load_reduction_pct=0.611610 moved_to_replicas=92708 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] avg=32768.0000 max=33097 max_rank=6 min=31710 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=49979 max_load_reduction_pct=0.601606 moved_to_replicas=97215 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32384, 36014, 464, 2805, 83076, 29983, 77413, 5] avg=32768.0000 max=83076 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.535278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] avg=32768.0000 max=33098 max_rank=0 min=30992 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30560 max_load_reduction_pct=0.480065 moved_to_replicas=73832 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[14, 8451, 63649, 32787, 21353, 68407, 32755, 34728] avg=32768.0000 max=68407 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.087616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32384, 36014, 464, 2805, 83076, 29983, 77413, 5] avg=32768.0000 max=83076 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.535278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] avg=32768.0000 max=33097 max_rank=6 min=31710 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=49979 max_load_reduction_pct=0.601606 moved_to_replicas=97215 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[58971, 13, 926, 32742, 63658, 33513, 22247, 50074] avg=32768.0000 max=63658 max_rank=4 min=13 min_rank=1 imbalance_ratio=1.942688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[32775, 33096, 33096, 32787, 31443, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=1 min=31443 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35311 max_load_reduction_pct=0.516190 moved_to_replicas=67496 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32384, 36014, 464, 2805, 83076, 29983, 77413, 5] avg=32768.0000 max=83076 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.535278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[58971, 13, 926, 32742, 63658, 33513, 22247, 50074] avg=32768.0000 max=63658 max_rank=4 min=13 min_rank=1 imbalance_ratio=1.942688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32384, 36014, 464, 2805, 83076, 29983, 77413, 5] avg=32768.0000 max=83076 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.535278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[97837, 32820, 32654, 371, 32741, 32818, 112, 32791] avg=32768.0000 max=97837 max_rank=0 min=112 min_rank=6 imbalance_ratio=2.985748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] avg=32768.0000 max=33097 max_rank=6 min=31710 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=49979 max_load_reduction_pct=0.601606 moved_to_replicas=97215 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] avg=32768.0000 max=33098 max_rank=0 min=30992 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30560 max_load_reduction_pct=0.480065 moved_to_replicas=73832 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] avg=32768.0000 max=33097 max_rank=6 min=31710 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=49979 max_load_reduction_pct=0.601606 moved_to_replicas=97215 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] avg=32768.0000 max=33098 max_rank=0 min=30992 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30560 max_load_reduction_pct=0.480065 moved_to_replicas=73832 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[58971, 13, 926, 32742, 63658, 33513, 22247, 50074] avg=32768.0000 max=63658 max_rank=4 min=13 min_rank=1 imbalance_ratio=1.942688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] avg=32768.0000 max=33097 max_rank=6 min=31710 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=49979 max_load_reduction_pct=0.601606 moved_to_replicas=97215 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] avg=32768.0000 max=33096 max_rank=0 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64741 max_load_reduction_pct=0.661723 moved_to_replicas=64741 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[8513, 59903, 46877, 85216, 43, 32769, 28808, 15] avg=32768.0000 max=85216 max_rank=3 min=15 min_rank=7 imbalance_ratio=2.600586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[58971, 13, 926, 32742, 63658, 33513, 22247, 50074] avg=32768.0000 max=63658 max_rank=4 min=13 min_rank=1 imbalance_ratio=1.942688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] avg=32768.0000 max=33098 max_rank=0 min=30992 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30560 max_load_reduction_pct=0.480065 moved_to_replicas=73832 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[97837, 32820, 32654, 371, 32741, 32818, 112, 32791] avg=32768.0000 max=97837 max_rank=0 min=112 min_rank=6 imbalance_ratio=2.985748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[33095, 33097, 33097, 33096, 33095, 32769, 31117, 32778] avg=32768.0000 max=33097 max_rank=1 min=31117 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=52119 max_load_reduction_pct=0.611610 moved_to_replicas=92708 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[58971, 13, 926, 32742, 63658, 33513, 22247, 50074] avg=32768.0000 max=63658 max_rank=4 min=13 min_rank=1 imbalance_ratio=1.942688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[97837, 32820, 32654, 371, 32741, 32818, 112, 32791] avg=32768.0000 max=97837 max_rank=0 min=112 min_rank=6 imbalance_ratio=2.985748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32772, 102034, 32598, 72, 58882, 35775, 0, 11] avg=32768.0000 max=102034 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.113831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[58971, 13, 926, 32742, 63658, 33513, 22247, 50074] avg=32768.0000 max=63658 max_rank=4 min=13 min_rank=1 imbalance_ratio=1.942688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] avg=32768.0000 max=33098 max_rank=0 min=30992 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30560 max_load_reduction_pct=0.480065 moved_to_replicas=73832 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] avg=32768.0000 max=33096 max_rank=0 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64741 max_load_reduction_pct=0.661723 moved_to_replicas=64741 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] avg=32768.0000 max=33098 max_rank=0 min=30992 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30560 max_load_reduction_pct=0.480065 moved_to_replicas=73832 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] avg=32768.0000 max=33096 max_rank=0 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64741 max_load_reduction_pct=0.661723 moved_to_replicas=64741 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] avg=32768.0000 max=33097 max_rank=1 min=32042 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68937 max_load_reduction_pct=0.675628 moved_to_replicas=97403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[97837, 32820, 32654, 371, 32741, 32818, 112, 32791] avg=32768.0000 max=97837 max_rank=0 min=112 min_rank=6 imbalance_ratio=2.985748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] avg=32768.0000 max=33098 max_rank=0 min=30992 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30560 max_load_reduction_pct=0.480065 moved_to_replicas=73832 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32384, 36014, 464, 2805, 83076, 29983, 77413, 5] avg=32768.0000 max=83076 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.535278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[97837, 32820, 32654, 371, 32741, 32818, 112, 32791] avg=32768.0000 max=97837 max_rank=0 min=112 min_rank=6 imbalance_ratio=2.985748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] avg=32768.0000 max=33096 max_rank=0 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64741 max_load_reduction_pct=0.661723 moved_to_replicas=64741 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32772, 102034, 32598, 72, 58882, 35775, 0, 11] avg=32768.0000 max=102034 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.113831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[97837, 32820, 32654, 371, 32741, 32818, 112, 32791] avg=32768.0000 max=97837 max_rank=0 min=112 min_rank=6 imbalance_ratio=2.985748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32772, 102034, 32598, 72, 58882, 35775, 0, 11] avg=32768.0000 max=102034 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.113831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32384, 33096, 31710, 33096, 33096, 32901, 33097, 32764] avg=32768.0000 max=33097 max_rank=6 min=31710 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=49979 max_load_reduction_pct=0.601606 moved_to_replicas=97215 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[11, 0, 71426, 98230, 689, 65490, 18380, 7918] avg=32768.0000 max=98230 max_rank=3 min=0 min_rank=1 imbalance_ratio=2.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] avg=32768.0000 max=33096 max_rank=0 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64741 max_load_reduction_pct=0.661723 moved_to_replicas=64741 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[97837, 32820, 32654, 371, 32741, 32818, 112, 32791] avg=32768.0000 max=97837 max_rank=0 min=112 min_rank=6 imbalance_ratio=2.985748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] avg=32768.0000 max=33097 max_rank=1 min=32042 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68937 max_load_reduction_pct=0.675628 moved_to_replicas=97403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] avg=32768.0000 max=33096 max_rank=0 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64741 max_load_reduction_pct=0.661723 moved_to_replicas=64741 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] avg=32768.0000 max=33097 max_rank=1 min=32042 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68937 max_load_reduction_pct=0.675628 moved_to_replicas=97403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] avg=32768.0000 max=33098 max_rank=7 min=31518 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65132 max_load_reduction_pct=0.663056 moved_to_replicas=135858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32772, 102034, 32598, 72, 58882, 35775, 0, 11] avg=32768.0000 max=102034 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.113831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] avg=32768.0000 max=33096 max_rank=0 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64741 max_load_reduction_pct=0.661723 moved_to_replicas=64741 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[58971, 13, 926, 32742, 63658, 33513, 22247, 50074] avg=32768.0000 max=63658 max_rank=4 min=13 min_rank=1 imbalance_ratio=1.942688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32772, 102034, 32598, 72, 58882, 35775, 0, 11] avg=32768.0000 max=102034 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.113831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] avg=32768.0000 max=33097 max_rank=1 min=32042 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68937 max_load_reduction_pct=0.675628 moved_to_replicas=97403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[11, 0, 71426, 98230, 689, 65490, 18380, 7918] avg=32768.0000 max=98230 max_rank=3 min=0 min_rank=1 imbalance_ratio=2.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32772, 102034, 32598, 72, 58882, 35775, 0, 11] avg=32768.0000 max=102034 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.113831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[11, 0, 71426, 98230, 689, 65490, 18380, 7918] avg=32768.0000 max=98230 max_rank=3 min=0 min_rank=1 imbalance_ratio=2.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32112, 33498, 32773, 32765] avg=32768.0000 max=33498 max_rank=5 min=32112 min_rank=4 imbalance_ratio=1.022278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[33098, 30992, 32929, 32742, 33096, 33096, 33096, 33095] avg=32768.0000 max=33098 max_rank=0 min=30992 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=30560 max_load_reduction_pct=0.480065 moved_to_replicas=73832 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] avg=32768.0000 max=33097 max_rank=1 min=32042 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68937 max_load_reduction_pct=0.675628 moved_to_replicas=97403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32772, 102034, 32598, 72, 58882, 35775, 0, 11] avg=32768.0000 max=102034 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.113831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] avg=32768.0000 max=33098 max_rank=7 min=31518 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65132 max_load_reduction_pct=0.663056 moved_to_replicas=135858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] avg=32768.0000 max=33097 max_rank=1 min=32042 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68937 max_load_reduction_pct=0.675628 moved_to_replicas=97403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] avg=32768.0000 max=33096 max_rank=5 min=32514 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=402 max_load_reduction_pct=0.012001 moved_to_replicas=402 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] avg=32768.0000 max=33098 max_rank=7 min=31518 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65132 max_load_reduction_pct=0.663056 moved_to_replicas=135858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[11, 0, 71426, 98230, 689, 65490, 18380, 7918] avg=32768.0000 max=98230 max_rank=3 min=0 min_rank=1 imbalance_ratio=2.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] avg=32768.0000 max=33097 max_rank=1 min=32042 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68937 max_load_reduction_pct=0.675628 moved_to_replicas=97403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[97837, 32820, 32654, 371, 32741, 32818, 112, 32791] avg=32768.0000 max=97837 max_rank=0 min=112 min_rank=6 imbalance_ratio=2.985748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[11, 0, 71426, 98230, 689, 65490, 18380, 7918] avg=32768.0000 max=98230 max_rank=3 min=0 min_rank=1 imbalance_ratio=2.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] avg=32768.0000 max=33098 max_rank=7 min=31518 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65132 max_load_reduction_pct=0.663056 moved_to_replicas=135858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32112, 33498, 32773, 32765] avg=32768.0000 max=33498 max_rank=5 min=32112 min_rank=4 imbalance_ratio=1.022278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[11, 0, 71426, 98230, 689, 65490, 18380, 7918] avg=32768.0000 max=98230 max_rank=3 min=0 min_rank=1 imbalance_ratio=2.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[703, 62955, 32795, 65199, 12, 67695, 32764, 21] avg=32768.0000 max=67695 max_rank=5 min=12 min_rank=4 imbalance_ratio=2.065887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32112, 33498, 32773, 32765] avg=32768.0000 max=33498 max_rank=5 min=32112 min_rank=4 imbalance_ratio=1.022278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32820, 32654, 32349, 32741, 32818, 32875, 32791] avg=32768.0000 max=33096 max_rank=0 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64741 max_load_reduction_pct=0.661723 moved_to_replicas=64741 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] avg=32768.0000 max=33098 max_rank=7 min=31518 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65132 max_load_reduction_pct=0.663056 moved_to_replicas=135858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[11, 0, 71426, 98230, 689, 65490, 18380, 7918] avg=32768.0000 max=98230 max_rank=3 min=0 min_rank=1 imbalance_ratio=2.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] avg=32768.0000 max=33096 max_rank=5 min=32514 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=402 max_load_reduction_pct=0.012001 moved_to_replicas=402 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] avg=32768.0000 max=33098 max_rank=7 min=31518 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65132 max_load_reduction_pct=0.663056 moved_to_replicas=135858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] avg=32768.0000 max=33096 max_rank=1 min=31787 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34599 max_load_reduction_pct=0.511101 moved_to_replicas=96561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32112, 33498, 32773, 32765] avg=32768.0000 max=33498 max_rank=5 min=32112 min_rank=4 imbalance_ratio=1.022278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] avg=32768.0000 max=33096 max_rank=5 min=32514 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=402 max_load_reduction_pct=0.012001 moved_to_replicas=402 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] avg=32768.0000 max=33098 max_rank=7 min=31518 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65132 max_load_reduction_pct=0.663056 moved_to_replicas=135858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32772, 102034, 32598, 72, 58882, 35775, 0, 11] avg=32768.0000 max=102034 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.113831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32112, 33498, 32773, 32765] avg=32768.0000 max=33498 max_rank=5 min=32112 min_rank=4 imbalance_ratio=1.022278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[703, 62955, 32795, 65199, 12, 67695, 32764, 21] avg=32768.0000 max=67695 max_rank=5 min=12 min_rank=4 imbalance_ratio=2.065887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] avg=32768.0000 max=33096 max_rank=5 min=32514 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=402 max_load_reduction_pct=0.012001 moved_to_replicas=402 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32112, 33498, 32773, 32765] avg=32768.0000 max=33498 max_rank=5 min=32112 min_rank=4 imbalance_ratio=1.022278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[2, 31110, 93562, 15311, 24046, 32259, 32772, 33082] avg=32768.0000 max=93562 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.855286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[703, 62955, 32795, 65199, 12, 67695, 32764, 21] avg=32768.0000 max=67695 max_rank=5 min=12 min_rank=4 imbalance_ratio=2.065887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32772, 33097, 32598, 32042, 33095, 33096, 32763, 32681] avg=32768.0000 max=33097 max_rank=1 min=32042 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=68937 max_load_reduction_pct=0.675628 moved_to_replicas=97403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32112, 33498, 32773, 32765] avg=32768.0000 max=33498 max_rank=5 min=32112 min_rank=4 imbalance_ratio=1.022278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] avg=32768.0000 max=33096 max_rank=5 min=32514 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=402 max_load_reduction_pct=0.012001 moved_to_replicas=402 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] avg=32768.0000 max=33096 max_rank=1 min=31787 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34599 max_load_reduction_pct=0.511101 moved_to_replicas=96561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] avg=32768.0000 max=33096 max_rank=5 min=32514 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=402 max_load_reduction_pct=0.012001 moved_to_replicas=402 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] avg=32768.0000 max=33097 max_rank=3 min=32007 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=60465 max_load_reduction_pct=0.646256 moved_to_replicas=60466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] avg=32768.0000 max=33096 max_rank=1 min=31787 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34599 max_load_reduction_pct=0.511101 moved_to_replicas=96561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[703, 62955, 32795, 65199, 12, 67695, 32764, 21] avg=32768.0000 max=67695 max_rank=5 min=12 min_rank=4 imbalance_ratio=2.065887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] avg=32768.0000 max=33096 max_rank=5 min=32514 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=402 max_load_reduction_pct=0.012001 moved_to_replicas=402 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[11, 0, 71426, 98230, 689, 65490, 18380, 7918] avg=32768.0000 max=98230 max_rank=3 min=0 min_rank=1 imbalance_ratio=2.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[2, 31110, 93562, 15311, 24046, 32259, 32772, 33082] avg=32768.0000 max=93562 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.855286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[703, 62955, 32795, 65199, 12, 67695, 32764, 21] avg=32768.0000 max=67695 max_rank=5 min=12 min_rank=4 imbalance_ratio=2.065887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] avg=32768.0000 max=33096 max_rank=1 min=31787 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34599 max_load_reduction_pct=0.511101 moved_to_replicas=96561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[703, 62955, 32795, 65199, 12, 67695, 32764, 21] avg=32768.0000 max=67695 max_rank=5 min=12 min_rank=4 imbalance_ratio=2.065887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[8, 32188, 55462, 40300, 9, 0, 35865, 98312] avg=32768.0000 max=98312 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.000244 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[2, 31110, 93562, 15311, 24046, 32259, 32772, 33082] avg=32768.0000 max=93562 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.855286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[32382, 32763, 33095, 33096, 33096, 33096, 31518, 33098] avg=32768.0000 max=33098 max_rank=7 min=31518 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=65132 max_load_reduction_pct=0.663056 moved_to_replicas=135858 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[703, 62955, 32795, 65199, 12, 67695, 32764, 21] avg=32768.0000 max=67695 max_rank=5 min=12 min_rank=4 imbalance_ratio=2.065887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] avg=32768.0000 max=33097 max_rank=3 min=32007 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=60465 max_load_reduction_pct=0.646256 moved_to_replicas=60466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] avg=32768.0000 max=33096 max_rank=1 min=31787 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34599 max_load_reduction_pct=0.511101 moved_to_replicas=96561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] avg=32768.0000 max=33096 max_rank=1 min=31787 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34599 max_load_reduction_pct=0.511101 moved_to_replicas=96561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65216 max_load_reduction_pct=0.663357 moved_to_replicas=97555 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] avg=32768.0000 max=33097 max_rank=3 min=32007 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=60465 max_load_reduction_pct=0.646256 moved_to_replicas=60466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[2, 31110, 93562, 15311, 24046, 32259, 32772, 33082] avg=32768.0000 max=93562 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.855286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] avg=32768.0000 max=33096 max_rank=1 min=31787 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34599 max_load_reduction_pct=0.511101 moved_to_replicas=96561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32112, 33498, 32773, 32765] avg=32768.0000 max=33498 max_rank=5 min=32112 min_rank=4 imbalance_ratio=1.022278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[8, 32188, 55462, 40300, 9, 0, 35865, 98312] avg=32768.0000 max=98312 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.000244 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[2, 31110, 93562, 15311, 24046, 32259, 32772, 33082] avg=32768.0000 max=93562 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.855286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] avg=32768.0000 max=33097 max_rank=3 min=32007 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=60465 max_load_reduction_pct=0.646256 moved_to_replicas=60466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[62, 91369, 7, 134, 67657, 26299, 32304, 44312] avg=32768.0000 max=91369 max_rank=1 min=7 min_rank=2 imbalance_ratio=2.788361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[2, 31110, 93562, 15311, 24046, 32259, 32772, 33082] avg=32768.0000 max=93562 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.855286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[8, 32188, 55462, 40300, 9, 0, 35865, 98312] avg=32768.0000 max=98312 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.000244 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[32768, 32801, 32719, 32708, 32514, 33096, 32773, 32765] avg=32768.0000 max=33096 max_rank=5 min=32514 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=402 max_load_reduction_pct=0.012001 moved_to_replicas=402 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[2, 31110, 93562, 15311, 24046, 32259, 32772, 33082] avg=32768.0000 max=93562 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.855286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65216 max_load_reduction_pct=0.663357 moved_to_replicas=97555 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] avg=32768.0000 max=33097 max_rank=3 min=32007 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=60465 max_load_reduction_pct=0.646256 moved_to_replicas=60466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] avg=32768.0000 max=33096 max_rank=4 min=31791 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=58273 max_load_reduction_pct=0.637776 moved_to_replicas=104050 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] avg=32768.0000 max=33097 max_rank=3 min=32007 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=60465 max_load_reduction_pct=0.646256 moved_to_replicas=60466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65216 max_load_reduction_pct=0.663357 moved_to_replicas=97555 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[8, 32188, 55462, 40300, 9, 0, 35865, 98312] avg=32768.0000 max=98312 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.000244 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] avg=32768.0000 max=33097 max_rank=3 min=32007 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=60465 max_load_reduction_pct=0.646256 moved_to_replicas=60466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[703, 62955, 32795, 65199, 12, 67695, 32764, 21] avg=32768.0000 max=67695 max_rank=5 min=12 min_rank=4 imbalance_ratio=2.065887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[62, 91369, 7, 134, 67657, 26299, 32304, 44312] avg=32768.0000 max=91369 max_rank=1 min=7 min_rank=2 imbalance_ratio=2.788361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65216 max_load_reduction_pct=0.663357 moved_to_replicas=97555 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[8, 32188, 55462, 40300, 9, 0, 35865, 98312] avg=32768.0000 max=98312 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.000244 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[5, 33220, 32764, 65115, 97576, 660, 32792, 12] avg=32768.0000 max=97576 max_rank=4 min=5 min_rank=0 imbalance_ratio=2.977783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[8, 32188, 55462, 40300, 9, 0, 35865, 98312] avg=32768.0000 max=98312 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.000244 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[62, 91369, 7, 134, 67657, 26299, 32304, 44312] avg=32768.0000 max=91369 max_rank=1 min=7 min_rank=2 imbalance_ratio=2.788361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[8, 32188, 55462, 40300, 9, 0, 35865, 98312] avg=32768.0000 max=98312 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.000244 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[32806, 33096, 32795, 33096, 31787, 33096, 32764, 32704] avg=32768.0000 max=33096 max_rank=1 min=31787 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=34599 max_load_reduction_pct=0.511101 moved_to_replicas=96561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] avg=32768.0000 max=33096 max_rank=4 min=31791 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=58273 max_load_reduction_pct=0.637776 moved_to_replicas=104050 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65216 max_load_reduction_pct=0.663357 moved_to_replicas=97555 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64480 max_load_reduction_pct=0.660818 moved_to_replicas=96623 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65216 max_load_reduction_pct=0.663357 moved_to_replicas=97555 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] avg=32768.0000 max=33096 max_rank=4 min=31791 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=58273 max_load_reduction_pct=0.637776 moved_to_replicas=104050 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65216 max_load_reduction_pct=0.663357 moved_to_replicas=97555 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[62, 91369, 7, 134, 67657, 26299, 32304, 44312] avg=32768.0000 max=91369 max_rank=1 min=7 min_rank=2 imbalance_ratio=2.788361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[2, 31110, 93562, 15311, 24046, 32259, 32772, 33082] avg=32768.0000 max=93562 max_rank=2 min=2 min_rank=0 imbalance_ratio=2.855286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[5, 33220, 32764, 65115, 97576, 660, 32792, 12] avg=32768.0000 max=97576 max_rank=4 min=5 min_rank=0 imbalance_ratio=2.977783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[32755, 54748, 31586, 9509, 67997, 10, 32763, 32776] avg=32768.0000 max=67997 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.075104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[62, 91369, 7, 134, 67657, 26299, 32304, 44312] avg=32768.0000 max=91369 max_rank=1 min=7 min_rank=2 imbalance_ratio=2.788361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[5, 33220, 32764, 65115, 97576, 660, 32792, 12] avg=32768.0000 max=97576 max_rank=4 min=5 min_rank=0 imbalance_ratio=2.977783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[62, 91369, 7, 134, 67657, 26299, 32304, 44312] avg=32768.0000 max=91369 max_rank=1 min=7 min_rank=2 imbalance_ratio=2.788361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] avg=32768.0000 max=33096 max_rank=4 min=31791 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=58273 max_load_reduction_pct=0.637776 moved_to_replicas=104050 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64480 max_load_reduction_pct=0.660818 moved_to_replicas=96623 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[62, 91369, 7, 134, 67657, 26299, 32304, 44312] avg=32768.0000 max=91369 max_rank=1 min=7 min_rank=2 imbalance_ratio=2.788361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[32736, 32007, 33096, 33097, 33095, 32259, 32772, 33082] avg=32768.0000 max=33097 max_rank=3 min=32007 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=60465 max_load_reduction_pct=0.646256 moved_to_replicas=60466 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] avg=32768.0000 max=33096 max_rank=1 min=31789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34901 max_load_reduction_pct=0.513273 moved_to_replicas=56553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] avg=32768.0000 max=33096 max_rank=4 min=31791 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=58273 max_load_reduction_pct=0.637776 moved_to_replicas=104050 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64480 max_load_reduction_pct=0.660818 moved_to_replicas=96623 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] avg=32768.0000 max=33096 max_rank=4 min=31791 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=58273 max_load_reduction_pct=0.637776 moved_to_replicas=104050 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] avg=32768.0000 max=33096 max_rank=4 min=31791 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=58273 max_load_reduction_pct=0.637776 moved_to_replicas=104050 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[5, 33220, 32764, 65115, 97576, 660, 32792, 12] avg=32768.0000 max=97576 max_rank=4 min=5 min_rank=0 imbalance_ratio=2.977783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[32755, 54748, 31586, 9509, 67997, 10, 32763, 32776] avg=32768.0000 max=67997 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.075104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[8, 32188, 55462, 40300, 9, 0, 35865, 98312] avg=32768.0000 max=98312 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.000244 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[13, 41578, 31476, 60599, 736, 65401, 29449, 32892] avg=32768.0000 max=65401 max_rank=5 min=13 min_rank=0 imbalance_ratio=1.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64480 max_load_reduction_pct=0.660818 moved_to_replicas=96623 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[32755, 54748, 31586, 9509, 67997, 10, 32763, 32776] avg=32768.0000 max=67997 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.075104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[5, 33220, 32764, 65115, 97576, 660, 32792, 12] avg=32768.0000 max=97576 max_rank=4 min=5 min_rank=0 imbalance_ratio=2.977783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[5, 33220, 32764, 65115, 97576, 660, 32792, 12] avg=32768.0000 max=97576 max_rank=4 min=5 min_rank=0 imbalance_ratio=2.977783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] avg=32768.0000 max=33096 max_rank=1 min=31789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34901 max_load_reduction_pct=0.513273 moved_to_replicas=56553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[5, 33220, 32764, 65115, 97576, 660, 32792, 12] avg=32768.0000 max=97576 max_rank=4 min=5 min_rank=0 imbalance_ratio=2.977783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32461, 32188, 33095, 33096, 32349, 32763, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65216 max_load_reduction_pct=0.663357 moved_to_replicas=97555 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] avg=32768.0000 max=33098 max_rank=4 min=31476 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32303 max_load_reduction_pct=0.493922 moved_to_replicas=68290 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] avg=32768.0000 max=33096 max_rank=1 min=31789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34901 max_load_reduction_pct=0.513273 moved_to_replicas=56553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64480 max_load_reduction_pct=0.660818 moved_to_replicas=96623 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64480 max_load_reduction_pct=0.660818 moved_to_replicas=96623 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[32755, 54748, 31586, 9509, 67997, 10, 32763, 32776] avg=32768.0000 max=67997 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.075104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64480 max_load_reduction_pct=0.660818 moved_to_replicas=96623 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[13, 41578, 31476, 60599, 736, 65401, 29449, 32892] avg=32768.0000 max=65401 max_rank=5 min=13 min_rank=0 imbalance_ratio=1.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[62, 91369, 7, 134, 67657, 26299, 32304, 44312] avg=32768.0000 max=91369 max_rank=1 min=7 min_rank=2 imbalance_ratio=2.788361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 5236, 65465, 90194, 4, 14454, 21939] avg=32768.0000 max=90194 max_rank=4 min=4 min_rank=5 imbalance_ratio=2.752502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] avg=32768.0000 max=33096 max_rank=1 min=31789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34901 max_load_reduction_pct=0.513273 moved_to_replicas=56553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[13, 41578, 31476, 60599, 736, 65401, 29449, 32892] avg=32768.0000 max=65401 max_rank=5 min=13 min_rank=0 imbalance_ratio=1.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[32755, 54748, 31586, 9509, 67997, 10, 32763, 32776] avg=32768.0000 max=67997 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.075104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[32755, 54748, 31586, 9509, 67997, 10, 32763, 32776] avg=32768.0000 max=67997 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.075104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] avg=32768.0000 max=33098 max_rank=4 min=31476 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32303 max_load_reduction_pct=0.493922 moved_to_replicas=68290 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[32755, 54748, 31586, 9509, 67997, 10, 32763, 32776] avg=32768.0000 max=67997 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.075104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[31791, 33095, 32770, 32897, 33096, 33095, 32304, 33096] avg=32768.0000 max=33096 max_rank=4 min=31791 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=58273 max_load_reduction_pct=0.637776 moved_to_replicas=104050 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32084 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57098 max_load_reduction_pct=0.633058 moved_to_replicas=89467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] avg=32768.0000 max=33098 max_rank=4 min=31476 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32303 max_load_reduction_pct=0.493922 moved_to_replicas=68290 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] avg=32768.0000 max=33096 max_rank=1 min=31789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34901 max_load_reduction_pct=0.513273 moved_to_replicas=56553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] avg=32768.0000 max=33096 max_rank=1 min=31789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34901 max_load_reduction_pct=0.513273 moved_to_replicas=56553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[13, 41578, 31476, 60599, 736, 65401, 29449, 32892] avg=32768.0000 max=65401 max_rank=5 min=13 min_rank=0 imbalance_ratio=1.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] avg=32768.0000 max=33096 max_rank=1 min=31789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34901 max_load_reduction_pct=0.513273 moved_to_replicas=56553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 5236, 65465, 90194, 4, 14454, 21939] avg=32768.0000 max=90194 max_rank=4 min=4 min_rank=5 imbalance_ratio=2.752502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=26 tokens_per_ep_rank=[51210, 63878, 32620, 2, 1, 65808, 22851, 25774] avg=32768.0000 max=65808 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[5, 33220, 32764, 65115, 97576, 660, 32792, 12] avg=32768.0000 max=97576 max_rank=4 min=5 min_rank=0 imbalance_ratio=2.977783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] avg=32768.0000 max=33098 max_rank=4 min=31476 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32303 max_load_reduction_pct=0.493922 moved_to_replicas=68290 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 5236, 65465, 90194, 4, 14454, 21939] avg=32768.0000 max=90194 max_rank=4 min=4 min_rank=5 imbalance_ratio=2.752502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[13, 41578, 31476, 60599, 736, 65401, 29449, 32892] avg=32768.0000 max=65401 max_rank=5 min=13 min_rank=0 imbalance_ratio=1.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[13, 41578, 31476, 60599, 736, 65401, 29449, 32892] avg=32768.0000 max=65401 max_rank=5 min=13 min_rank=0 imbalance_ratio=1.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32084 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57098 max_load_reduction_pct=0.633058 moved_to_replicas=89467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[13, 41578, 31476, 60599, 736, 65401, 29449, 32892] avg=32768.0000 max=65401 max_rank=5 min=13 min_rank=0 imbalance_ratio=1.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=26 tokens_per_ep_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] avg=32768.0000 max=33097 max_rank=0 min=31331 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32711 max_load_reduction_pct=0.497067 moved_to_replicas=81608 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[32768, 33096, 32764, 33096, 33096, 32679, 32792, 31853] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64480 max_load_reduction_pct=0.660818 moved_to_replicas=96623 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32084 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57098 max_load_reduction_pct=0.633058 moved_to_replicas=89467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] avg=32768.0000 max=33098 max_rank=4 min=31476 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32303 max_load_reduction_pct=0.493922 moved_to_replicas=68290 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] avg=32768.0000 max=33098 max_rank=4 min=31476 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32303 max_load_reduction_pct=0.493922 moved_to_replicas=68290 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 5236, 65465, 90194, 4, 14454, 21939] avg=32768.0000 max=90194 max_rank=4 min=4 min_rank=5 imbalance_ratio=2.752502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] avg=32768.0000 max=33098 max_rank=4 min=31476 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32303 max_load_reduction_pct=0.493922 moved_to_replicas=68290 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=26 tokens_per_ep_rank=[51210, 63878, 32620, 2, 1, 65808, 22851, 25774] avg=32768.0000 max=65808 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=27 tokens_per_ep_rank=[1004, 65395, 309, 553, 32770, 25087, 72553, 64473] avg=32768.0000 max=72553 max_rank=6 min=309 min_rank=2 imbalance_ratio=2.214142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=26 tokens_per_ep_rank=[51210, 63878, 32620, 2, 1, 65808, 22851, 25774] avg=32768.0000 max=65808 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[32755, 54748, 31586, 9509, 67997, 10, 32763, 32776] avg=32768.0000 max=67997 max_rank=4 min=10 min_rank=5 imbalance_ratio=2.075104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32084 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57098 max_load_reduction_pct=0.633058 moved_to_replicas=89467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 5236, 65465, 90194, 4, 14454, 21939] avg=32768.0000 max=90194 max_rank=4 min=4 min_rank=5 imbalance_ratio=2.752502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=26 tokens_per_ep_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] avg=32768.0000 max=33097 max_rank=0 min=31331 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32711 max_load_reduction_pct=0.497067 moved_to_replicas=81608 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 5236, 65465, 90194, 4, 14454, 21939] avg=32768.0000 max=90194 max_rank=4 min=4 min_rank=5 imbalance_ratio=2.752502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=27 tokens_per_ep_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30830 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39456 max_load_reduction_pct=0.543823 moved_to_replicas=103133 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 5236, 65465, 90194, 4, 14454, 21939] avg=32768.0000 max=90194 max_rank=4 min=4 min_rank=5 imbalance_ratio=2.752502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=26 tokens_per_ep_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] avg=32768.0000 max=33097 max_rank=0 min=31331 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32711 max_load_reduction_pct=0.497067 moved_to_replicas=81608 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[32755, 33096, 31789, 33096, 33096, 32773, 32763, 32776] avg=32768.0000 max=33096 max_rank=1 min=31789 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=34901 max_load_reduction_pct=0.513273 moved_to_replicas=56553 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32084 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57098 max_load_reduction_pct=0.633058 moved_to_replicas=89467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32084 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57098 max_load_reduction_pct=0.633058 moved_to_replicas=89467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=26 tokens_per_ep_rank=[51210, 63878, 32620, 2, 1, 65808, 22851, 25774] avg=32768.0000 max=65808 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32084 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57098 max_load_reduction_pct=0.633058 moved_to_replicas=89467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=27 tokens_per_ep_rank=[1004, 65395, 309, 553, 32770, 25087, 72553, 64473] avg=32768.0000 max=72553 max_rank=6 min=309 min_rank=2 imbalance_ratio=2.214142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=28 tokens_per_ep_rank=[48120, 32763, 32802, 32767, 32765, 50290, 4, 32633] avg=32768.0000 max=50290 max_rank=5 min=4 min_rank=6 imbalance_ratio=1.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=27 tokens_per_ep_rank=[1004, 65395, 309, 553, 32770, 25087, 72553, 64473] avg=32768.0000 max=72553 max_rank=6 min=309 min_rank=2 imbalance_ratio=2.214142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=26 tokens_per_ep_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] avg=32768.0000 max=33097 max_rank=0 min=31331 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32711 max_load_reduction_pct=0.497067 moved_to_replicas=81608 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[13, 41578, 31476, 60599, 736, 65401, 29449, 32892] avg=32768.0000 max=65401 max_rank=5 min=13 min_rank=0 imbalance_ratio=1.995880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=26 tokens_per_ep_rank=[51210, 63878, 32620, 2, 1, 65808, 22851, 25774] avg=32768.0000 max=65808 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=27 tokens_per_ep_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30830 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39456 max_load_reduction_pct=0.543823 moved_to_replicas=103133 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=26 tokens_per_ep_rank=[51210, 63878, 32620, 2, 1, 65808, 22851, 25774] avg=32768.0000 max=65808 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=28 tokens_per_ep_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] avg=32768.0000 max=33097 max_rank=5 min=32223 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17193 max_load_reduction_pct=0.341877 moved_to_replicas=32218 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=26 tokens_per_ep_rank=[51210, 63878, 32620, 2, 1, 65808, 22851, 25774] avg=32768.0000 max=65808 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=27 tokens_per_ep_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30830 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39456 max_load_reduction_pct=0.543823 moved_to_replicas=103133 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=26 tokens_per_ep_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] avg=32768.0000 max=33097 max_rank=0 min=31331 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32711 max_load_reduction_pct=0.497067 moved_to_replicas=81608 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[32318, 33095, 31476, 33095, 33098, 33096, 33074, 32892] avg=32768.0000 max=33098 max_rank=4 min=31476 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32303 max_load_reduction_pct=0.493922 moved_to_replicas=68290 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=27 tokens_per_ep_rank=[1004, 65395, 309, 553, 32770, 25087, 72553, 64473] avg=32768.0000 max=72553 max_rank=6 min=309 min_rank=2 imbalance_ratio=2.214142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=26 tokens_per_ep_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] avg=32768.0000 max=33097 max_rank=0 min=31331 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32711 max_load_reduction_pct=0.497067 moved_to_replicas=81608 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=26 tokens_per_ep_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] avg=32768.0000 max=33097 max_rank=0 min=31331 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32711 max_load_reduction_pct=0.497067 moved_to_replicas=81608 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=28 tokens_per_ep_rank=[48120, 32763, 32802, 32767, 32765, 50290, 4, 32633] avg=32768.0000 max=50290 max_rank=5 min=4 min_rank=6 imbalance_ratio=1.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=29 tokens_per_ep_rank=[347, 65588, 32641, 5, 41417, 59696, 151, 62299] avg=32768.0000 max=65588 max_rank=1 min=5 min_rank=3 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=28 tokens_per_ep_rank=[48120, 32763, 32802, 32767, 32765, 50290, 4, 32633] avg=32768.0000 max=50290 max_rank=5 min=4 min_rank=6 imbalance_ratio=1.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=27 tokens_per_ep_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30830 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39456 max_load_reduction_pct=0.543823 moved_to_replicas=103133 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=27 tokens_per_ep_rank=[1004, 65395, 309, 553, 32770, 25087, 72553, 64473] avg=32768.0000 max=72553 max_rank=6 min=309 min_rank=2 imbalance_ratio=2.214142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 5236, 65465, 90194, 4, 14454, 21939] avg=32768.0000 max=90194 max_rank=4 min=4 min_rank=5 imbalance_ratio=2.752502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=28 tokens_per_ep_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] avg=32768.0000 max=33097 max_rank=5 min=32223 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17193 max_load_reduction_pct=0.341877 moved_to_replicas=32218 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=27 tokens_per_ep_rank=[1004, 65395, 309, 553, 32770, 25087, 72553, 64473] avg=32768.0000 max=72553 max_rank=6 min=309 min_rank=2 imbalance_ratio=2.214142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=29 tokens_per_ep_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] avg=32768.0000 max=33098 max_rank=4 min=31526 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32490 max_load_reduction_pct=0.495365 moved_to_replicas=96616 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=27 tokens_per_ep_rank=[1004, 65395, 309, 553, 32770, 25087, 72553, 64473] avg=32768.0000 max=72553 max_rank=6 min=309 min_rank=2 imbalance_ratio=2.214142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=28 tokens_per_ep_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] avg=32768.0000 max=33097 max_rank=5 min=32223 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17193 max_load_reduction_pct=0.341877 moved_to_replicas=32218 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=27 tokens_per_ep_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30830 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39456 max_load_reduction_pct=0.543823 moved_to_replicas=103133 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=25 tokens_per_ep_rank=[32084, 32768, 32142, 33096, 33095, 32767, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32084 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57098 max_load_reduction_pct=0.633058 moved_to_replicas=89467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=28 tokens_per_ep_rank=[48120, 32763, 32802, 32767, 32765, 50290, 4, 32633] avg=32768.0000 max=50290 max_rank=5 min=4 min_rank=6 imbalance_ratio=1.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=27 tokens_per_ep_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30830 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39456 max_load_reduction_pct=0.543823 moved_to_replicas=103133 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=27 tokens_per_ep_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30830 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39456 max_load_reduction_pct=0.543823 moved_to_replicas=103133 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=29 tokens_per_ep_rank=[347, 65588, 32641, 5, 41417, 59696, 151, 62299] avg=32768.0000 max=65588 max_rank=1 min=5 min_rank=3 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=30 tokens_per_ep_rank=[33884, 98285, 4, 31044, 11032, 64741, 22864, 290] avg=32768.0000 max=98285 max_rank=1 min=4 min_rank=2 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=29 tokens_per_ep_rank=[347, 65588, 32641, 5, 41417, 59696, 151, 62299] avg=32768.0000 max=65588 max_rank=1 min=5 min_rank=3 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=28 tokens_per_ep_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] avg=32768.0000 max=33097 max_rank=5 min=32223 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17193 max_load_reduction_pct=0.341877 moved_to_replicas=32218 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=28 tokens_per_ep_rank=[48120, 32763, 32802, 32767, 32765, 50290, 4, 32633] avg=32768.0000 max=50290 max_rank=5 min=4 min_rank=6 imbalance_ratio=1.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=26 tokens_per_ep_rank=[51210, 63878, 32620, 2, 1, 65808, 22851, 25774] avg=32768.0000 max=65808 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=29 tokens_per_ep_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] avg=32768.0000 max=33098 max_rank=4 min=31526 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32490 max_load_reduction_pct=0.495365 moved_to_replicas=96616 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=29 tokens_per_ep_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] avg=32768.0000 max=33098 max_rank=4 min=31526 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32490 max_load_reduction_pct=0.495365 moved_to_replicas=96616 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] avg=32768.0000 max=33097 max_rank=4 min=31832 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65188 max_load_reduction_pct=0.663255 moved_to_replicas=97622 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=28 tokens_per_ep_rank=[48120, 32763, 32802, 32767, 32765, 50290, 4, 32633] avg=32768.0000 max=50290 max_rank=5 min=4 min_rank=6 imbalance_ratio=1.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=28 tokens_per_ep_rank=[48120, 32763, 32802, 32767, 32765, 50290, 4, 32633] avg=32768.0000 max=50290 max_rank=5 min=4 min_rank=6 imbalance_ratio=1.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=28 tokens_per_ep_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] avg=32768.0000 max=33097 max_rank=5 min=32223 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17193 max_load_reduction_pct=0.341877 moved_to_replicas=32218 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=26 tokens_per_ep_rank=[33097, 33096, 32620, 31331, 32713, 33096, 33094, 33097] avg=32768.0000 max=33097 max_rank=0 min=31331 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32711 max_load_reduction_pct=0.497067 moved_to_replicas=81608 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=29 tokens_per_ep_rank=[347, 65588, 32641, 5, 41417, 59696, 151, 62299] avg=32768.0000 max=65588 max_rank=1 min=5 min_rank=3 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=28 tokens_per_ep_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] avg=32768.0000 max=33097 max_rank=5 min=32223 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17193 max_load_reduction_pct=0.341877 moved_to_replicas=32218 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=30 tokens_per_ep_rank=[33884, 98285, 4, 31044, 11032, 64741, 22864, 290] avg=32768.0000 max=98285 max_rank=1 min=4 min_rank=2 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=28 tokens_per_ep_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] avg=32768.0000 max=33097 max_rank=5 min=32223 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17193 max_load_reduction_pct=0.341877 moved_to_replicas=32218 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=30 tokens_per_ep_rank=[33884, 98285, 4, 31044, 11032, 64741, 22864, 290] avg=32768.0000 max=98285 max_rank=1 min=4 min_rank=2 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=31 tokens_per_ep_rank=[12, 92053, 245, 4, 62516, 7859, 32764, 66691] avg=32768.0000 max=92053 max_rank=1 min=4 min_rank=3 imbalance_ratio=2.809235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=29 tokens_per_ep_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] avg=32768.0000 max=33098 max_rank=4 min=31526 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32490 max_load_reduction_pct=0.495365 moved_to_replicas=96616 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=29 tokens_per_ep_rank=[347, 65588, 32641, 5, 41417, 59696, 151, 62299] avg=32768.0000 max=65588 max_rank=1 min=5 min_rank=3 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] avg=32768.0000 max=33097 max_rank=4 min=31832 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65188 max_load_reduction_pct=0.663255 moved_to_replicas=97622 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=27 tokens_per_ep_rank=[1004, 65395, 309, 553, 32770, 25087, 72553, 64473] avg=32768.0000 max=72553 max_rank=6 min=309 min_rank=2 imbalance_ratio=2.214142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] avg=32768.0000 max=33097 max_rank=4 min=31832 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65188 max_load_reduction_pct=0.663255 moved_to_replicas=97622 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=31 tokens_per_ep_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] avg=32768.0000 max=33097 max_rank=5 min=31221 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=58956 max_load_reduction_pct=0.640457 moved_to_replicas=121972 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=29 tokens_per_ep_rank=[347, 65588, 32641, 5, 41417, 59696, 151, 62299] avg=32768.0000 max=65588 max_rank=1 min=5 min_rank=3 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=29 tokens_per_ep_rank=[347, 65588, 32641, 5, 41417, 59696, 151, 62299] avg=32768.0000 max=65588 max_rank=1 min=5 min_rank=3 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=29 tokens_per_ep_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] avg=32768.0000 max=33098 max_rank=4 min=31526 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32490 max_load_reduction_pct=0.495365 moved_to_replicas=96616 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=27 tokens_per_ep_rank=[33096, 33096, 33064, 33095, 32770, 30830, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30830 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=39456 max_load_reduction_pct=0.543823 moved_to_replicas=103133 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=30 tokens_per_ep_rank=[33884, 98285, 4, 31044, 11032, 64741, 22864, 290] avg=32768.0000 max=98285 max_rank=1 min=4 min_rank=2 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=31 tokens_per_ep_rank=[12, 92053, 245, 4, 62516, 7859, 32764, 66691] avg=32768.0000 max=92053 max_rank=1 min=4 min_rank=3 imbalance_ratio=2.809235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=29 tokens_per_ep_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] avg=32768.0000 max=33098 max_rank=4 min=31526 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32490 max_load_reduction_pct=0.495365 moved_to_replicas=96616 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=29 tokens_per_ep_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] avg=32768.0000 max=33098 max_rank=4 min=31526 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32490 max_load_reduction_pct=0.495365 moved_to_replicas=96616 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=31 tokens_per_ep_rank=[12, 92053, 245, 4, 62516, 7859, 32764, 66691] avg=32768.0000 max=92053 max_rank=1 min=4 min_rank=3 imbalance_ratio=2.809235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=32 tokens_per_ep_rank=[7, 4, 32748, 65532, 348, 98225, 32516, 32764] avg=32768.0000 max=98225 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] avg=32768.0000 max=33097 max_rank=4 min=31832 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65188 max_load_reduction_pct=0.663255 moved_to_replicas=97622 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=31 tokens_per_ep_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] avg=32768.0000 max=33097 max_rank=5 min=31221 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=58956 max_load_reduction_pct=0.640457 moved_to_replicas=121972 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=30 tokens_per_ep_rank=[33884, 98285, 4, 31044, 11032, 64741, 22864, 290] avg=32768.0000 max=98285 max_rank=1 min=4 min_rank=2 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=28 tokens_per_ep_rank=[48120, 32763, 32802, 32767, 32765, 50290, 4, 32633] avg=32768.0000 max=50290 max_rank=5 min=4 min_rank=6 imbalance_ratio=1.534729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=31 tokens_per_ep_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] avg=32768.0000 max=33097 max_rank=5 min=31221 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=58956 max_load_reduction_pct=0.640457 moved_to_replicas=121972 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=32 tokens_per_ep_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] avg=32768.0000 max=33096 max_rank=3 min=32373 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=97565 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=30 tokens_per_ep_rank=[33884, 98285, 4, 31044, 11032, 64741, 22864, 290] avg=32768.0000 max=98285 max_rank=1 min=4 min_rank=2 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=30 tokens_per_ep_rank=[33884, 98285, 4, 31044, 11032, 64741, 22864, 290] avg=32768.0000 max=98285 max_rank=1 min=4 min_rank=2 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=31 tokens_per_ep_rank=[12, 92053, 245, 4, 62516, 7859, 32764, 66691] avg=32768.0000 max=92053 max_rank=1 min=4 min_rank=3 imbalance_ratio=2.809235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] avg=32768.0000 max=33097 max_rank=4 min=31832 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65188 max_load_reduction_pct=0.663255 moved_to_replicas=97622 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=28 tokens_per_ep_rank=[33094, 32763, 32802, 32767, 32765, 33097, 32223, 32633] avg=32768.0000 max=33097 max_rank=5 min=32223 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17193 max_load_reduction_pct=0.341877 moved_to_replicas=32218 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=32 tokens_per_ep_rank=[7, 4, 32748, 65532, 348, 98225, 32516, 32764] avg=32768.0000 max=98225 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] avg=32768.0000 max=33097 max_rank=4 min=31832 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65188 max_load_reduction_pct=0.663255 moved_to_replicas=97622 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] avg=32768.0000 max=33097 max_rank=4 min=31832 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65188 max_load_reduction_pct=0.663255 moved_to_replicas=97622 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=32 tokens_per_ep_rank=[7, 4, 32748, 65532, 348, 98225, 32516, 32764] avg=32768.0000 max=98225 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=33 tokens_per_ep_rank=[4, 64864, 33412, 74, 31756, 32728, 34323, 64983] avg=32768.0000 max=64983 max_rank=7 min=4 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=31 tokens_per_ep_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] avg=32768.0000 max=33097 max_rank=5 min=31221 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=58956 max_load_reduction_pct=0.640457 moved_to_replicas=121972 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=32 tokens_per_ep_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] avg=32768.0000 max=33096 max_rank=3 min=32373 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=97565 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=31 tokens_per_ep_rank=[12, 92053, 245, 4, 62516, 7859, 32764, 66691] avg=32768.0000 max=92053 max_rank=1 min=4 min_rank=3 imbalance_ratio=2.809235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=32 tokens_per_ep_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] avg=32768.0000 max=33096 max_rank=3 min=32373 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=97565 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=33 tokens_per_ep_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=29 tokens_per_ep_rank=[347, 65588, 32641, 5, 41417, 59696, 151, 62299] avg=32768.0000 max=65588 max_rank=1 min=5 min_rank=3 imbalance_ratio=2.001587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=31 tokens_per_ep_rank=[12, 92053, 245, 4, 62516, 7859, 32764, 66691] avg=32768.0000 max=92053 max_rank=1 min=4 min_rank=3 imbalance_ratio=2.809235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=31 tokens_per_ep_rank=[12, 92053, 245, 4, 62516, 7859, 32764, 66691] avg=32768.0000 max=92053 max_rank=1 min=4 min_rank=3 imbalance_ratio=2.809235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=32 tokens_per_ep_rank=[7, 4, 32748, 65532, 348, 98225, 32516, 32764] avg=32768.0000 max=98225 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=31 tokens_per_ep_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] avg=32768.0000 max=33097 max_rank=5 min=31221 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=58956 max_load_reduction_pct=0.640457 moved_to_replicas=121972 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=29 tokens_per_ep_rank=[33095, 33096, 32641, 32497, 33098, 33095, 31526, 33096] avg=32768.0000 max=33098 max_rank=4 min=31526 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32490 max_load_reduction_pct=0.495365 moved_to_replicas=96616 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=33 tokens_per_ep_rank=[4, 64864, 33412, 74, 31756, 32728, 34323, 64983] avg=32768.0000 max=64983 max_rank=7 min=4 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=31 tokens_per_ep_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] avg=32768.0000 max=33097 max_rank=5 min=31221 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=58956 max_load_reduction_pct=0.640457 moved_to_replicas=121972 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=33 tokens_per_ep_rank=[4, 64864, 33412, 74, 31756, 32728, 34323, 64983] avg=32768.0000 max=64983 max_rank=7 min=4 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=34 tokens_per_ep_rank=[36068, 98205, 10, 4, 31164, 11, 0, 96682] avg=32768.0000 max=98205 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=31 tokens_per_ep_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] avg=32768.0000 max=33097 max_rank=5 min=31221 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=58956 max_load_reduction_pct=0.640457 moved_to_replicas=121972 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=32 tokens_per_ep_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] avg=32768.0000 max=33096 max_rank=3 min=32373 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=97565 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=33 tokens_per_ep_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=32 tokens_per_ep_rank=[7, 4, 32748, 65532, 348, 98225, 32516, 32764] avg=32768.0000 max=98225 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=33 tokens_per_ep_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] avg=32768.0000 max=33096 max_rank=0 min=31874 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=131667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=30 tokens_per_ep_rank=[33884, 98285, 4, 31044, 11032, 64741, 22864, 290] avg=32768.0000 max=98285 max_rank=1 min=4 min_rank=2 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=32 tokens_per_ep_rank=[7, 4, 32748, 65532, 348, 98225, 32516, 32764] avg=32768.0000 max=98225 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=32 tokens_per_ep_rank=[7, 4, 32748, 65532, 348, 98225, 32516, 32764] avg=32768.0000 max=98225 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=33 tokens_per_ep_rank=[4, 64864, 33412, 74, 31756, 32728, 34323, 64983] avg=32768.0000 max=64983 max_rank=7 min=4 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=32 tokens_per_ep_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] avg=32768.0000 max=33096 max_rank=3 min=32373 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=97565 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=34 tokens_per_ep_rank=[36068, 98205, 10, 4, 31164, 11, 0, 96682] avg=32768.0000 max=98205 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=30 tokens_per_ep_rank=[33096, 33096, 32767, 31832, 33097, 33096, 32444, 32716] avg=32768.0000 max=33097 max_rank=4 min=31832 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65188 max_load_reduction_pct=0.663255 moved_to_replicas=97622 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=34 tokens_per_ep_rank=[36068, 98205, 10, 4, 31164, 11, 0, 96682] avg=32768.0000 max=98205 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=32 tokens_per_ep_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] avg=32768.0000 max=33096 max_rank=3 min=32373 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=97565 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=35 tokens_per_ep_rank=[8, 4, 16, 98291, 32764, 64332, 66729, 0] avg=32768.0000 max=98291 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=33 tokens_per_ep_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=32 tokens_per_ep_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] avg=32768.0000 max=33096 max_rank=3 min=32373 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=97565 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] avg=32768.0000 max=33096 max_rank=0 min=31874 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=131667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=33 tokens_per_ep_rank=[4, 64864, 33412, 74, 31756, 32728, 34323, 64983] avg=32768.0000 max=64983 max_rank=7 min=4 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] avg=32768.0000 max=33096 max_rank=0 min=31874 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=131667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=35 tokens_per_ep_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=3 min=32123 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65195 max_load_reduction_pct=0.663286 moved_to_replicas=130064 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=31 tokens_per_ep_rank=[12, 92053, 245, 4, 62516, 7859, 32764, 66691] avg=32768.0000 max=92053 max_rank=1 min=4 min_rank=3 imbalance_ratio=2.809235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=33 tokens_per_ep_rank=[4, 64864, 33412, 74, 31756, 32728, 34323, 64983] avg=32768.0000 max=64983 max_rank=7 min=4 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=34 tokens_per_ep_rank=[36068, 98205, 10, 4, 31164, 11, 0, 96682] avg=32768.0000 max=98205 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=33 tokens_per_ep_rank=[4, 64864, 33412, 74, 31756, 32728, 34323, 64983] avg=32768.0000 max=64983 max_rank=7 min=4 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=33 tokens_per_ep_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=35 tokens_per_ep_rank=[8, 4, 16, 98291, 32764, 64332, 66729, 0] avg=32768.0000 max=98291 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=31 tokens_per_ep_rank=[31221, 33095, 33008, 32767, 33096, 33097, 32764, 33096] avg=32768.0000 max=33097 max_rank=5 min=31221 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=58956 max_load_reduction_pct=0.640457 moved_to_replicas=121972 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=33 tokens_per_ep_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=36 tokens_per_ep_rank=[130731, 94, 4, 65136, 4, 32746, 33424, 5] avg=32768.0000 max=130731 max_rank=0 min=4 min_rank=2 imbalance_ratio=3.989594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=35 tokens_per_ep_rank=[8, 4, 16, 98291, 32764, 64332, 66729, 0] avg=32768.0000 max=98291 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] avg=32768.0000 max=33096 max_rank=0 min=31874 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=131667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=33 tokens_per_ep_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=35 tokens_per_ep_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=3 min=32123 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65195 max_load_reduction_pct=0.663286 moved_to_replicas=130064 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=34 tokens_per_ep_rank=[36068, 98205, 10, 4, 31164, 11, 0, 96682] avg=32768.0000 max=98205 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] avg=32768.0000 max=33096 max_rank=0 min=32134 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97635 max_load_reduction_pct=0.746839 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=35 tokens_per_ep_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=3 min=32123 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65195 max_load_reduction_pct=0.663286 moved_to_replicas=130064 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=32 tokens_per_ep_rank=[7, 4, 32748, 65532, 348, 98225, 32516, 32764] avg=32768.0000 max=98225 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.997589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=34 tokens_per_ep_rank=[36068, 98205, 10, 4, 31164, 11, 0, 96682] avg=32768.0000 max=98205 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=35 tokens_per_ep_rank=[8, 4, 16, 98291, 32764, 64332, 66729, 0] avg=32768.0000 max=98291 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=34 tokens_per_ep_rank=[36068, 98205, 10, 4, 31164, 11, 0, 96682] avg=32768.0000 max=98205 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] avg=32768.0000 max=33096 max_rank=0 min=31874 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=131667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=36 tokens_per_ep_rank=[130731, 94, 4, 65136, 4, 32746, 33424, 5] avg=32768.0000 max=130731 max_rank=0 min=4 min_rank=2 imbalance_ratio=3.989594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] avg=32768.0000 max=33096 max_rank=0 min=31874 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=131667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=32 tokens_per_ep_rank=[32373, 32767, 32748, 33096, 32784, 33096, 32516, 32764] avg=32768.0000 max=33096 max_rank=3 min=32373 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65129 max_load_reduction_pct=0.663059 moved_to_replicas=97565 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=37 tokens_per_ep_rank=[43322, 10, 12, 0, 65495, 0, 87689, 65616] avg=32768.0000 max=87689 max_rank=6 min=0 min_rank=3 imbalance_ratio=2.676056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=36 tokens_per_ep_rank=[130731, 94, 4, 65136, 4, 32746, 33424, 5] avg=32768.0000 max=130731 max_rank=0 min=4 min_rank=2 imbalance_ratio=3.989594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=35 tokens_per_ep_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=3 min=32123 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65195 max_load_reduction_pct=0.663286 moved_to_replicas=130064 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] avg=32768.0000 max=33096 max_rank=0 min=31874 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=131667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] avg=32768.0000 max=33096 max_rank=0 min=32134 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97635 max_load_reduction_pct=0.746839 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=35 tokens_per_ep_rank=[8, 4, 16, 98291, 32764, 64332, 66729, 0] avg=32768.0000 max=98291 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=37 tokens_per_ep_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32057 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=54593 max_load_reduction_pct=0.622575 moved_to_replicas=129738 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] avg=32768.0000 max=33096 max_rank=0 min=32134 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97635 max_load_reduction_pct=0.746839 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=35 tokens_per_ep_rank=[8, 4, 16, 98291, 32764, 64332, 66729, 0] avg=32768.0000 max=98291 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=33 tokens_per_ep_rank=[4, 64864, 33412, 74, 31756, 32728, 34323, 64983] avg=32768.0000 max=64983 max_rank=7 min=4 min_rank=0 imbalance_ratio=1.983124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=36 tokens_per_ep_rank=[130731, 94, 4, 65136, 4, 32746, 33424, 5] avg=32768.0000 max=130731 max_rank=0 min=4 min_rank=2 imbalance_ratio=3.989594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=35 tokens_per_ep_rank=[8, 4, 16, 98291, 32764, 64332, 66729, 0] avg=32768.0000 max=98291 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=37 tokens_per_ep_rank=[43322, 10, 12, 0, 65495, 0, 87689, 65616] avg=32768.0000 max=87689 max_rank=6 min=0 min_rank=3 imbalance_ratio=2.676056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=35 tokens_per_ep_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=3 min=32123 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65195 max_load_reduction_pct=0.663286 moved_to_replicas=130064 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=35 tokens_per_ep_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=3 min=32123 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65195 max_load_reduction_pct=0.663286 moved_to_replicas=130064 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=38 tokens_per_ep_rank=[35142, 32700, 106, 32762, 18, 63056, 32755, 65605] avg=32768.0000 max=65605 max_rank=7 min=18 min_rank=4 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=33 tokens_per_ep_rank=[31891, 33096, 33096, 32158, 32983, 32728, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31887 max_load_reduction_pct=0.490698 moved_to_replicas=65198 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=37 tokens_per_ep_rank=[43322, 10, 12, 0, 65495, 0, 87689, 65616] avg=32768.0000 max=87689 max_rank=6 min=0 min_rank=3 imbalance_ratio=2.676056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] avg=32768.0000 max=33096 max_rank=0 min=32134 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97635 max_load_reduction_pct=0.746839 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=37 tokens_per_ep_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32057 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=54593 max_load_reduction_pct=0.622575 moved_to_replicas=129738 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=35 tokens_per_ep_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=3 min=32123 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65195 max_load_reduction_pct=0.663286 moved_to_replicas=130064 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=38 tokens_per_ep_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=0 min=32112 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64515 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=37 tokens_per_ep_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32057 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=54593 max_load_reduction_pct=0.622575 moved_to_replicas=129738 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=36 tokens_per_ep_rank=[130731, 94, 4, 65136, 4, 32746, 33424, 5] avg=32768.0000 max=130731 max_rank=0 min=4 min_rank=2 imbalance_ratio=3.989594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=36 tokens_per_ep_rank=[130731, 94, 4, 65136, 4, 32746, 33424, 5] avg=32768.0000 max=130731 max_rank=0 min=4 min_rank=2 imbalance_ratio=3.989594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=34 tokens_per_ep_rank=[36068, 98205, 10, 4, 31164, 11, 0, 96682] avg=32768.0000 max=98205 max_rank=1 min=0 min_rank=6 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=37 tokens_per_ep_rank=[43322, 10, 12, 0, 65495, 0, 87689, 65616] avg=32768.0000 max=87689 max_rank=6 min=0 min_rank=3 imbalance_ratio=2.676056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=38 tokens_per_ep_rank=[35142, 32700, 106, 32762, 18, 63056, 32755, 65605] avg=32768.0000 max=65605 max_rank=7 min=18 min_rank=4 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=36 tokens_per_ep_rank=[130731, 94, 4, 65136, 4, 32746, 33424, 5] avg=32768.0000 max=130731 max_rank=0 min=4 min_rank=2 imbalance_ratio=3.989594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] avg=32768.0000 max=33096 max_rank=0 min=32134 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97635 max_load_reduction_pct=0.746839 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] avg=32768.0000 max=33096 max_rank=0 min=32134 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97635 max_load_reduction_pct=0.746839 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=39 tokens_per_ep_rank=[19411, 0, 46332, 65284, 8, 65528, 32831, 32750] avg=32768.0000 max=65528 max_rank=5 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=38 tokens_per_ep_rank=[35142, 32700, 106, 32762, 18, 63056, 32755, 65605] avg=32768.0000 max=65605 max_rank=7 min=18 min_rank=4 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=34 tokens_per_ep_rank=[33096, 33096, 32773, 32357, 31874, 33096, 32756, 33096] avg=32768.0000 max=33096 max_rank=0 min=31874 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65109 max_load_reduction_pct=0.662991 moved_to_replicas=131667 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=37 tokens_per_ep_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32057 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=54593 max_load_reduction_pct=0.622575 moved_to_replicas=129738 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=38 tokens_per_ep_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=0 min=32112 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64515 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] avg=32768.0000 max=33096 max_rank=0 min=32134 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97635 max_load_reduction_pct=0.746839 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=39 tokens_per_ep_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] avg=32768.0000 max=33096 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=77856 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=38 tokens_per_ep_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=0 min=32112 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64515 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=37 tokens_per_ep_rank=[43322, 10, 12, 0, 65495, 0, 87689, 65616] avg=32768.0000 max=87689 max_rank=6 min=0 min_rank=3 imbalance_ratio=2.676056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=37 tokens_per_ep_rank=[43322, 10, 12, 0, 65495, 0, 87689, 65616] avg=32768.0000 max=87689 max_rank=6 min=0 min_rank=3 imbalance_ratio=2.676056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=35 tokens_per_ep_rank=[8, 4, 16, 98291, 32764, 64332, 66729, 0] avg=32768.0000 max=98291 max_rank=3 min=0 min_rank=7 imbalance_ratio=2.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=38 tokens_per_ep_rank=[35142, 32700, 106, 32762, 18, 63056, 32755, 65605] avg=32768.0000 max=65605 max_rank=7 min=18 min_rank=4 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=39 tokens_per_ep_rank=[19411, 0, 46332, 65284, 8, 65528, 32831, 32750] avg=32768.0000 max=65528 max_rank=5 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=37 tokens_per_ep_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32057 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=54593 max_load_reduction_pct=0.622575 moved_to_replicas=129738 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=40 tokens_per_ep_rank=[65474, 37145, 32760, 32809, 32566, 32684, 28702, 4] avg=32768.0000 max=65474 max_rank=0 min=4 min_rank=7 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=37 tokens_per_ep_rank=[43322, 10, 12, 0, 65495, 0, 87689, 65616] avg=32768.0000 max=87689 max_rank=6 min=0 min_rank=3 imbalance_ratio=2.676056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=39 tokens_per_ep_rank=[19411, 0, 46332, 65284, 8, 65528, 32831, 32750] avg=32768.0000 max=65528 max_rank=5 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=37 tokens_per_ep_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32057 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=54593 max_load_reduction_pct=0.622575 moved_to_replicas=129738 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=35 tokens_per_ep_rank=[32770, 32436, 32123, 33096, 32764, 33096, 33096, 32763] avg=32768.0000 max=33096 max_rank=3 min=32123 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65195 max_load_reduction_pct=0.663286 moved_to_replicas=130064 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=38 tokens_per_ep_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=0 min=32112 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64515 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=39 tokens_per_ep_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] avg=32768.0000 max=33096 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=77856 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=40 tokens_per_ep_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=36427 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=39 tokens_per_ep_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] avg=32768.0000 max=33096 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=77856 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=37 tokens_per_ep_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32057 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=54593 max_load_reduction_pct=0.622575 moved_to_replicas=129738 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=38 tokens_per_ep_rank=[35142, 32700, 106, 32762, 18, 63056, 32755, 65605] avg=32768.0000 max=65605 max_rank=7 min=18 min_rank=4 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=38 tokens_per_ep_rank=[35142, 32700, 106, 32762, 18, 63056, 32755, 65605] avg=32768.0000 max=65605 max_rank=7 min=18 min_rank=4 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=36 tokens_per_ep_rank=[130731, 94, 4, 65136, 4, 32746, 33424, 5] avg=32768.0000 max=130731 max_rank=0 min=4 min_rank=2 imbalance_ratio=3.989594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=39 tokens_per_ep_rank=[19411, 0, 46332, 65284, 8, 65528, 32831, 32750] avg=32768.0000 max=65528 max_rank=5 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=40 tokens_per_ep_rank=[65474, 37145, 32760, 32809, 32566, 32684, 28702, 4] avg=32768.0000 max=65474 max_rank=0 min=4 min_rank=7 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=38 tokens_per_ep_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=0 min=32112 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64515 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=41 tokens_per_ep_rank=[4, 4, 62010, 44147, 31137, 38663, 61830, 24349] avg=32768.0000 max=62010 max_rank=2 min=4 min_rank=0 imbalance_ratio=1.892395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=38 tokens_per_ep_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=0 min=32112 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64515 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=40 tokens_per_ep_rank=[65474, 37145, 32760, 32809, 32566, 32684, 28702, 4] avg=32768.0000 max=65474 max_rank=0 min=4 min_rank=7 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=38 tokens_per_ep_rank=[35142, 32700, 106, 32762, 18, 63056, 32755, 65605] avg=32768.0000 max=65605 max_rank=7 min=18 min_rank=4 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=39 tokens_per_ep_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] avg=32768.0000 max=33096 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=77856 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=36 tokens_per_ep_rank=[33096, 32134, 32767, 33096, 32767, 32746, 33096, 32442] avg=32768.0000 max=33096 max_rank=0 min=32134 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97635 max_load_reduction_pct=0.746839 moved_to_replicas=130003 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=40 tokens_per_ep_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=36427 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=41 tokens_per_ep_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31137 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=28912 max_load_reduction_pct=0.466247 moved_to_replicas=74266 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=40 tokens_per_ep_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=36427 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=38 tokens_per_ep_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=0 min=32112 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64515 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=39 tokens_per_ep_rank=[19411, 0, 46332, 65284, 8, 65528, 32831, 32750] avg=32768.0000 max=65528 max_rank=5 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=39 tokens_per_ep_rank=[19411, 0, 46332, 65284, 8, 65528, 32831, 32750] avg=32768.0000 max=65528 max_rank=5 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=40 tokens_per_ep_rank=[65474, 37145, 32760, 32809, 32566, 32684, 28702, 4] avg=32768.0000 max=65474 max_rank=0 min=4 min_rank=7 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=41 tokens_per_ep_rank=[4, 4, 62010, 44147, 31137, 38663, 61830, 24349] avg=32768.0000 max=62010 max_rank=2 min=4 min_rank=0 imbalance_ratio=1.892395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=37 tokens_per_ep_rank=[43322, 10, 12, 0, 65495, 0, 87689, 65616] avg=32768.0000 max=87689 max_rank=6 min=0 min_rank=3 imbalance_ratio=2.676056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=39 tokens_per_ep_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] avg=32768.0000 max=33096 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=77856 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=42 tokens_per_ep_rank=[32810, 40526, 30961, 67449, 1122, 64814, 24459, 3] avg=32768.0000 max=67449 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.058380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=39 tokens_per_ep_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] avg=32768.0000 max=33096 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=77856 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=41 tokens_per_ep_rank=[4, 4, 62010, 44147, 31137, 38663, 61830, 24349] avg=32768.0000 max=62010 max_rank=2 min=4 min_rank=0 imbalance_ratio=1.892395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=39 tokens_per_ep_rank=[19411, 0, 46332, 65284, 8, 65528, 32831, 32750] avg=32768.0000 max=65528 max_rank=5 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=40 tokens_per_ep_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=36427 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=37 tokens_per_ep_rank=[33095, 32530, 32411, 32763, 33096, 32057, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32057 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=54593 max_load_reduction_pct=0.622575 moved_to_replicas=129738 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=41 tokens_per_ep_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31137 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=28912 max_load_reduction_pct=0.466247 moved_to_replicas=74266 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=42 tokens_per_ep_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] avg=32768.0000 max=33097 max_rank=1 min=31088 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=34352 max_load_reduction_pct=0.509303 moved_to_replicas=73501 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=41 tokens_per_ep_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31137 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=28912 max_load_reduction_pct=0.466247 moved_to_replicas=74266 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=39 tokens_per_ep_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] avg=32768.0000 max=33096 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=77856 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=40 tokens_per_ep_rank=[65474, 37145, 32760, 32809, 32566, 32684, 28702, 4] avg=32768.0000 max=65474 max_rank=0 min=4 min_rank=7 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=40 tokens_per_ep_rank=[65474, 37145, 32760, 32809, 32566, 32684, 28702, 4] avg=32768.0000 max=65474 max_rank=0 min=4 min_rank=7 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=41 tokens_per_ep_rank=[4, 4, 62010, 44147, 31137, 38663, 61830, 24349] avg=32768.0000 max=62010 max_rank=2 min=4 min_rank=0 imbalance_ratio=1.892395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=38 tokens_per_ep_rank=[35142, 32700, 106, 32762, 18, 63056, 32755, 65605] avg=32768.0000 max=65605 max_rank=7 min=18 min_rank=4 imbalance_ratio=2.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=42 tokens_per_ep_rank=[32810, 40526, 30961, 67449, 1122, 64814, 24459, 3] avg=32768.0000 max=67449 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.058380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=40 tokens_per_ep_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=36427 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 64, 23547, 14472, 38426, 65514, 54595] avg=32768.0000 max=65514 max_rank=6 min=64 min_rank=2 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=40 tokens_per_ep_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=36427 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=42 tokens_per_ep_rank=[32810, 40526, 30961, 67449, 1122, 64814, 24459, 3] avg=32768.0000 max=67449 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.058380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=40 tokens_per_ep_rank=[65474, 37145, 32760, 32809, 32566, 32684, 28702, 4] avg=32768.0000 max=65474 max_rank=0 min=4 min_rank=7 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=41 tokens_per_ep_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31137 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=28912 max_load_reduction_pct=0.466247 moved_to_replicas=74266 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=42 tokens_per_ep_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] avg=32768.0000 max=33097 max_rank=1 min=31088 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=34352 max_load_reduction_pct=0.509303 moved_to_replicas=73501 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=38 tokens_per_ep_rank=[33096, 32700, 32112, 32762, 32527, 33096, 32755, 33096] avg=32768.0000 max=33096 max_rank=0 min=32112 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32509 max_load_reduction_pct=0.495526 moved_to_replicas=64515 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31753 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32417 max_load_reduction_pct=0.494810 moved_to_replicas=59247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=42 tokens_per_ep_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] avg=32768.0000 max=33097 max_rank=1 min=31088 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=34352 max_load_reduction_pct=0.509303 moved_to_replicas=73501 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=41 tokens_per_ep_rank=[4, 4, 62010, 44147, 31137, 38663, 61830, 24349] avg=32768.0000 max=62010 max_rank=2 min=4 min_rank=0 imbalance_ratio=1.892395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=40 tokens_per_ep_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=36427 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=41 tokens_per_ep_rank=[4, 4, 62010, 44147, 31137, 38663, 61830, 24349] avg=32768.0000 max=62010 max_rank=2 min=4 min_rank=0 imbalance_ratio=1.892395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=42 tokens_per_ep_rank=[32810, 40526, 30961, 67449, 1122, 64814, 24459, 3] avg=32768.0000 max=67449 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.058380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 64, 23547, 14472, 38426, 65514, 54595] avg=32768.0000 max=65514 max_rank=6 min=64 min_rank=2 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=41 tokens_per_ep_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31137 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=28912 max_load_reduction_pct=0.466247 moved_to_replicas=74266 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=39 tokens_per_ep_rank=[19411, 0, 46332, 65284, 8, 65528, 32831, 32750] avg=32768.0000 max=65528 max_rank=5 min=0 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=44 tokens_per_ep_rank=[250, 12, 99053, 32769, 7, 11, 97054, 32988] avg=32768.0000 max=99053 max_rank=2 min=7 min_rank=4 imbalance_ratio=3.022858 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=41 tokens_per_ep_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31137 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=28912 max_load_reduction_pct=0.466247 moved_to_replicas=74266 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 64, 23547, 14472, 38426, 65514, 54595] avg=32768.0000 max=65514 max_rank=6 min=64 min_rank=2 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=42 tokens_per_ep_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] avg=32768.0000 max=33097 max_rank=1 min=31088 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=34352 max_load_reduction_pct=0.509303 moved_to_replicas=73501 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=41 tokens_per_ep_rank=[4, 4, 62010, 44147, 31137, 38663, 61830, 24349] avg=32768.0000 max=62010 max_rank=2 min=4 min_rank=0 imbalance_ratio=1.892395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31753 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32417 max_load_reduction_pct=0.494810 moved_to_replicas=59247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=39 tokens_per_ep_rank=[32649, 32432, 33094, 33096, 32196, 33096, 32831, 32750] avg=32768.0000 max=33096 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=77856 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=44 tokens_per_ep_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] avg=32768.0000 max=33096 max_rank=2 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65957 max_load_reduction_pct=0.665876 moved_to_replicas=129915 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31753 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32417 max_load_reduction_pct=0.494810 moved_to_replicas=59247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=42 tokens_per_ep_rank=[32810, 40526, 30961, 67449, 1122, 64814, 24459, 3] avg=32768.0000 max=67449 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.058380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=41 tokens_per_ep_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31137 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=28912 max_load_reduction_pct=0.466247 moved_to_replicas=74266 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=42 tokens_per_ep_rank=[32810, 40526, 30961, 67449, 1122, 64814, 24459, 3] avg=32768.0000 max=67449 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.058380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 64, 23547, 14472, 38426, 65514, 54595] avg=32768.0000 max=65514 max_rank=6 min=64 min_rank=2 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=44 tokens_per_ep_rank=[250, 12, 99053, 32769, 7, 11, 97054, 32988] avg=32768.0000 max=99053 max_rank=2 min=7 min_rank=4 imbalance_ratio=3.022858 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=42 tokens_per_ep_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] avg=32768.0000 max=33097 max_rank=1 min=31088 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=34352 max_load_reduction_pct=0.509303 moved_to_replicas=73501 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=40 tokens_per_ep_rank=[65474, 37145, 32760, 32809, 32566, 32684, 28702, 4] avg=32768.0000 max=65474 max_rank=0 min=4 min_rank=7 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=45 tokens_per_ep_rank=[32770, 65520, 98288, 4, 32784, 1, 4, 32773] avg=32768.0000 max=98288 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=44 tokens_per_ep_rank=[250, 12, 99053, 32769, 7, 11, 97054, 32988] avg=32768.0000 max=99053 max_rank=2 min=7 min_rank=4 imbalance_ratio=3.022858 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=42 tokens_per_ep_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] avg=32768.0000 max=33097 max_rank=1 min=31088 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=34352 max_load_reduction_pct=0.509303 moved_to_replicas=73501 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31753 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32417 max_load_reduction_pct=0.494810 moved_to_replicas=59247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=42 tokens_per_ep_rank=[32810, 40526, 30961, 67449, 1122, 64814, 24459, 3] avg=32768.0000 max=67449 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.058380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=44 tokens_per_ep_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] avg=32768.0000 max=33096 max_rank=2 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65957 max_load_reduction_pct=0.665876 moved_to_replicas=129915 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=45 tokens_per_ep_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] avg=32768.0000 max=33096 max_rank=1 min=32428 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65192 max_load_reduction_pct=0.663275 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=40 tokens_per_ep_rank=[33096, 33096, 32760, 32809, 32566, 32684, 32751, 32382] avg=32768.0000 max=33096 max_rank=0 min=32382 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=36427 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=44 tokens_per_ep_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] avg=32768.0000 max=33096 max_rank=2 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65957 max_load_reduction_pct=0.665876 moved_to_replicas=129915 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 64, 23547, 14472, 38426, 65514, 54595] avg=32768.0000 max=65514 max_rank=6 min=64 min_rank=2 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=42 tokens_per_ep_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] avg=32768.0000 max=33097 max_rank=1 min=31088 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=34352 max_load_reduction_pct=0.509303 moved_to_replicas=73501 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 64, 23547, 14472, 38426, 65514, 54595] avg=32768.0000 max=65514 max_rank=6 min=64 min_rank=2 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=44 tokens_per_ep_rank=[250, 12, 99053, 32769, 7, 11, 97054, 32988] avg=32768.0000 max=99053 max_rank=2 min=7 min_rank=4 imbalance_ratio=3.022858 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=45 tokens_per_ep_rank=[32770, 65520, 98288, 4, 32784, 1, 4, 32773] avg=32768.0000 max=98288 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31753 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32417 max_load_reduction_pct=0.494810 moved_to_replicas=59247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=46 tokens_per_ep_rank=[37866, 8, 34877, 32764, 8, 57979, 32768, 65874] avg=32768.0000 max=65874 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=45 tokens_per_ep_rank=[32770, 65520, 98288, 4, 32784, 1, 4, 32773] avg=32768.0000 max=98288 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31753 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32417 max_load_reduction_pct=0.494810 moved_to_replicas=59247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=41 tokens_per_ep_rank=[4, 4, 62010, 44147, 31137, 38663, 61830, 24349] avg=32768.0000 max=62010 max_rank=2 min=4 min_rank=0 imbalance_ratio=1.892395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=44 tokens_per_ep_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] avg=32768.0000 max=33096 max_rank=2 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65957 max_load_reduction_pct=0.665876 moved_to_replicas=129915 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=45 tokens_per_ep_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] avg=32768.0000 max=33096 max_rank=1 min=32428 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65192 max_load_reduction_pct=0.663275 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 64, 23547, 14472, 38426, 65514, 54595] avg=32768.0000 max=65514 max_rank=6 min=64 min_rank=2 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=46 tokens_per_ep_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] avg=32768.0000 max=33098 max_rank=0 min=31454 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32776 max_load_reduction_pct=0.497556 moved_to_replicas=64212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=45 tokens_per_ep_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] avg=32768.0000 max=33096 max_rank=1 min=32428 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65192 max_load_reduction_pct=0.663275 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=41 tokens_per_ep_rank=[33094, 32431, 33096, 33096, 31137, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31137 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=28912 max_load_reduction_pct=0.466247 moved_to_replicas=74266 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=44 tokens_per_ep_rank=[250, 12, 99053, 32769, 7, 11, 97054, 32988] avg=32768.0000 max=99053 max_rank=2 min=7 min_rank=4 imbalance_ratio=3.022858 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31753 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32417 max_load_reduction_pct=0.494810 moved_to_replicas=59247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=44 tokens_per_ep_rank=[250, 12, 99053, 32769, 7, 11, 97054, 32988] avg=32768.0000 max=99053 max_rank=2 min=7 min_rank=4 imbalance_ratio=3.022858 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=45 tokens_per_ep_rank=[32770, 65520, 98288, 4, 32784, 1, 4, 32773] avg=32768.0000 max=98288 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=46 tokens_per_ep_rank=[37866, 8, 34877, 32764, 8, 57979, 32768, 65874] avg=32768.0000 max=65874 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=44 tokens_per_ep_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] avg=32768.0000 max=33096 max_rank=2 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65957 max_load_reduction_pct=0.665876 moved_to_replicas=129915 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=47 tokens_per_ep_rank=[7804, 32785, 32768, 89256, 32764, 32768, 416, 33583] avg=32768.0000 max=89256 max_rank=3 min=416 min_rank=6 imbalance_ratio=2.723877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=46 tokens_per_ep_rank=[37866, 8, 34877, 32764, 8, 57979, 32768, 65874] avg=32768.0000 max=65874 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=44 tokens_per_ep_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] avg=32768.0000 max=33096 max_rank=2 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65957 max_load_reduction_pct=0.665876 moved_to_replicas=129915 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=45 tokens_per_ep_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] avg=32768.0000 max=33096 max_rank=1 min=32428 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65192 max_load_reduction_pct=0.663275 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=42 tokens_per_ep_rank=[32810, 40526, 30961, 67449, 1122, 64814, 24459, 3] avg=32768.0000 max=67449 max_rank=3 min=3 min_rank=7 imbalance_ratio=2.058380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=46 tokens_per_ep_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] avg=32768.0000 max=33098 max_rank=0 min=31454 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32776 max_load_reduction_pct=0.497556 moved_to_replicas=64212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=44 tokens_per_ep_rank=[250, 12, 99053, 32769, 7, 11, 97054, 32988] avg=32768.0000 max=99053 max_rank=2 min=7 min_rank=4 imbalance_ratio=3.022858 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=47 tokens_per_ep_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31770 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=56159 max_load_reduction_pct=0.629190 moved_to_replicas=56647 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=46 tokens_per_ep_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] avg=32768.0000 max=33098 max_rank=0 min=31454 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32776 max_load_reduction_pct=0.497556 moved_to_replicas=64212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=45 tokens_per_ep_rank=[32770, 65520, 98288, 4, 32784, 1, 4, 32773] avg=32768.0000 max=98288 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=42 tokens_per_ep_rank=[32810, 33097, 31088, 33096, 33096, 33096, 33095, 32766] avg=32768.0000 max=33097 max_rank=1 min=31088 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=34352 max_load_reduction_pct=0.509303 moved_to_replicas=73501 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=44 tokens_per_ep_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] avg=32768.0000 max=33096 max_rank=2 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65957 max_load_reduction_pct=0.665876 moved_to_replicas=129915 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=45 tokens_per_ep_rank=[32770, 65520, 98288, 4, 32784, 1, 4, 32773] avg=32768.0000 max=98288 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=46 tokens_per_ep_rank=[37866, 8, 34877, 32764, 8, 57979, 32768, 65874] avg=32768.0000 max=65874 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=47 tokens_per_ep_rank=[7804, 32785, 32768, 89256, 32764, 32768, 416, 33583] avg=32768.0000 max=89256 max_rank=3 min=416 min_rank=6 imbalance_ratio=2.723877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=45 tokens_per_ep_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] avg=32768.0000 max=33096 max_rank=1 min=32428 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65192 max_load_reduction_pct=0.663275 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=48 tokens_per_ep_rank=[65659, 65524, 950, 32780, 32768, 33242, 278, 30943] avg=32768.0000 max=65659 max_rank=0 min=278 min_rank=6 imbalance_ratio=2.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=47 tokens_per_ep_rank=[7804, 32785, 32768, 89256, 32764, 32768, 416, 33583] avg=32768.0000 max=89256 max_rank=3 min=416 min_rank=6 imbalance_ratio=2.723877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=45 tokens_per_ep_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] avg=32768.0000 max=33096 max_rank=1 min=32428 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65192 max_load_reduction_pct=0.663275 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=46 tokens_per_ep_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] avg=32768.0000 max=33098 max_rank=0 min=31454 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32776 max_load_reduction_pct=0.497556 moved_to_replicas=64212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 64, 23547, 14472, 38426, 65514, 54595] avg=32768.0000 max=65514 max_rank=6 min=64 min_rank=2 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=47 tokens_per_ep_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31770 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=56159 max_load_reduction_pct=0.629190 moved_to_replicas=56647 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=45 tokens_per_ep_rank=[32770, 65520, 98288, 4, 32784, 1, 4, 32773] avg=32768.0000 max=98288 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=48 tokens_per_ep_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] avg=32768.0000 max=33096 max_rank=0 min=31371 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32563 max_load_reduction_pct=0.495941 moved_to_replicas=65137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=47 tokens_per_ep_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31770 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=56159 max_load_reduction_pct=0.629190 moved_to_replicas=56647 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=46 tokens_per_ep_rank=[37866, 8, 34877, 32764, 8, 57979, 32768, 65874] avg=32768.0000 max=65874 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32760, 32482, 31753, 33095, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31753 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32417 max_load_reduction_pct=0.494810 moved_to_replicas=59247 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=45 tokens_per_ep_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] avg=32768.0000 max=33096 max_rank=1 min=32428 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65192 max_load_reduction_pct=0.663275 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=46 tokens_per_ep_rank=[37866, 8, 34877, 32764, 8, 57979, 32768, 65874] avg=32768.0000 max=65874 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=47 tokens_per_ep_rank=[7804, 32785, 32768, 89256, 32764, 32768, 416, 33583] avg=32768.0000 max=89256 max_rank=3 min=416 min_rank=6 imbalance_ratio=2.723877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=48 tokens_per_ep_rank=[65659, 65524, 950, 32780, 32768, 33242, 278, 30943] avg=32768.0000 max=65659 max_rank=0 min=278 min_rank=6 imbalance_ratio=2.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=46 tokens_per_ep_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] avg=32768.0000 max=33098 max_rank=0 min=31454 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32776 max_load_reduction_pct=0.497556 moved_to_replicas=64212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=48 tokens_per_ep_rank=[65659, 65524, 950, 32780, 32768, 33242, 278, 30943] avg=32768.0000 max=65659 max_rank=0 min=278 min_rank=6 imbalance_ratio=2.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=46 tokens_per_ep_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] avg=32768.0000 max=33098 max_rank=0 min=31454 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32776 max_load_reduction_pct=0.497556 moved_to_replicas=64212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=44 tokens_per_ep_rank=[250, 12, 99053, 32769, 7, 11, 97054, 32988] avg=32768.0000 max=99053 max_rank=2 min=7 min_rank=4 imbalance_ratio=3.022858 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=48 tokens_per_ep_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] avg=32768.0000 max=33096 max_rank=0 min=31371 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32563 max_load_reduction_pct=0.495941 moved_to_replicas=65137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=47 tokens_per_ep_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31770 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=56159 max_load_reduction_pct=0.629190 moved_to_replicas=56647 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=46 tokens_per_ep_rank=[37866, 8, 34877, 32764, 8, 57979, 32768, 65874] avg=32768.0000 max=65874 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=48 tokens_per_ep_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] avg=32768.0000 max=33096 max_rank=0 min=31371 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32563 max_load_reduction_pct=0.495941 moved_to_replicas=65137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=47 tokens_per_ep_rank=[7804, 32785, 32768, 89256, 32764, 32768, 416, 33583] avg=32768.0000 max=89256 max_rank=3 min=416 min_rank=6 imbalance_ratio=2.723877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=44 tokens_per_ep_rank=[33013, 31648, 33096, 32769, 32765, 32769, 33096, 32988] avg=32768.0000 max=33096 max_rank=2 min=31648 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65957 max_load_reduction_pct=0.665876 moved_to_replicas=129915 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=46 tokens_per_ep_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] avg=32768.0000 max=33098 max_rank=0 min=31454 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32776 max_load_reduction_pct=0.497556 moved_to_replicas=64212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=47 tokens_per_ep_rank=[7804, 32785, 32768, 89256, 32764, 32768, 416, 33583] avg=32768.0000 max=89256 max_rank=3 min=416 min_rank=6 imbalance_ratio=2.723877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=48 tokens_per_ep_rank=[65659, 65524, 950, 32780, 32768, 33242, 278, 30943] avg=32768.0000 max=65659 max_rank=0 min=278 min_rank=6 imbalance_ratio=2.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=47 tokens_per_ep_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31770 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=56159 max_load_reduction_pct=0.629190 moved_to_replicas=56647 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=47 tokens_per_ep_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31770 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=56159 max_load_reduction_pct=0.629190 moved_to_replicas=56647 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=45 tokens_per_ep_rank=[32770, 65520, 98288, 4, 32784, 1, 4, 32773] avg=32768.0000 max=98288 max_rank=2 min=1 min_rank=5 imbalance_ratio=2.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=48 tokens_per_ep_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] avg=32768.0000 max=33096 max_rank=0 min=31371 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32563 max_load_reduction_pct=0.495941 moved_to_replicas=65137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=47 tokens_per_ep_rank=[7804, 32785, 32768, 89256, 32764, 32768, 416, 33583] avg=32768.0000 max=89256 max_rank=3 min=416 min_rank=6 imbalance_ratio=2.723877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=48 tokens_per_ep_rank=[65659, 65524, 950, 32780, 32768, 33242, 278, 30943] avg=32768.0000 max=65659 max_rank=0 min=278 min_rank=6 imbalance_ratio=2.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=45 tokens_per_ep_rank=[32770, 33096, 33096, 32433, 32784, 32764, 32428, 32773] avg=32768.0000 max=33096 max_rank=1 min=32428 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65192 max_load_reduction_pct=0.663275 moved_to_replicas=97616 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=48 tokens_per_ep_rank=[65659, 65524, 950, 32780, 32768, 33242, 278, 30943] avg=32768.0000 max=65659 max_rank=0 min=278 min_rank=6 imbalance_ratio=2.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=47 tokens_per_ep_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31770 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=56159 max_load_reduction_pct=0.629190 moved_to_replicas=56647 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=48 tokens_per_ep_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] avg=32768.0000 max=33096 max_rank=0 min=31371 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32563 max_load_reduction_pct=0.495941 moved_to_replicas=65137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=48 tokens_per_ep_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] avg=32768.0000 max=33096 max_rank=0 min=31371 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32563 max_load_reduction_pct=0.495941 moved_to_replicas=65137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=46 tokens_per_ep_rank=[37866, 8, 34877, 32764, 8, 57979, 32768, 65874] avg=32768.0000 max=65874 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=48 tokens_per_ep_rank=[65659, 65524, 950, 32780, 32768, 33242, 278, 30943] avg=32768.0000 max=65659 max_rank=0 min=278 min_rank=6 imbalance_ratio=2.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=46 tokens_per_ep_rank=[33098, 32771, 33096, 32764, 31454, 33097, 32768, 33096] avg=32768.0000 max=33098 max_rank=0 min=31454 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=32776 max_load_reduction_pct=0.497556 moved_to_replicas=64212 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=48 tokens_per_ep_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] avg=32768.0000 max=33096 max_rank=0 min=31371 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32563 max_load_reduction_pct=0.495941 moved_to_replicas=65137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=47 tokens_per_ep_rank=[7804, 32785, 32768, 89256, 32764, 32768, 416, 33583] avg=32768.0000 max=89256 max_rank=3 min=416 min_rank=6 imbalance_ratio=2.723877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=47 tokens_per_ep_rank=[31770, 32785, 32768, 33097, 32764, 32768, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31770 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=56159 max_load_reduction_pct=0.629190 moved_to_replicas=56647 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=48 tokens_per_ep_rank=[65659, 65524, 950, 32780, 32768, 33242, 278, 30943] avg=32768.0000 max=65659 max_rank=0 min=278 min_rank=6 imbalance_ratio=2.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=48 tokens_per_ep_rank=[33096, 33096, 33096, 32780, 32768, 33096, 32841, 31371] avg=32768.0000 max=33096 max_rank=0 min=31371 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32563 max_load_reduction_pct=0.495941 moved_to_replicas=65137 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=48 total_expected_recv=1570522 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376960 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=48 total_expected_recv=1573206 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376832 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=48 total_expected_recv=1572877 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376576 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=48 total_expected_recv=1571136 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379136 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=48 total_expected_recv=1574541 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374784 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=48 total_expected_recv=1577271 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=370816 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=48 total_expected_recv=1570838 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377088 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=48 total_expected_recv=1572521 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376704 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 15:47:07] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 714.9 | throughput per GPU (TFLOP/s/GPU): 124.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.767684E+00 | loss scale: 1.0 | grad norm: 3.062 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +[after training is done] datetime: 2026-08-26 15:47:07 +[rank0]:[W826 15:47:08.429159667 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:47:09.860331359 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:47:09.872563752 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:47:09.873193374 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:47:09.873829541 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:47:09.874398067 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:47:09.941847166 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:47:10.399617594 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:47:11.279000 134192 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:47:11.280000 134192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 134285 closing signal SIGTERM +W0826 15:47:11.280000 134192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 134286 closing signal SIGTERM +W0826 15:47:11.281000 134192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 134287 closing signal SIGTERM +W0826 15:47:11.282000 134192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 134288 closing signal SIGTERM +W0826 15:47:11.283000 134192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 134289 closing signal SIGTERM +W0826 15:47:11.283000 134192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 134290 closing signal SIGTERM +W0826 15:47:11.284000 134192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 134291 closing signal SIGTERM +W0826 15:47:11.285000 134192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 134292 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -26006,4 +26014,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 49721 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 134192 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..da3d93881060ca888acc942c7caa32a0e2d584f6 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,240 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 993 +1, 1015 +2, 1027 +3, 1025 +4, 1003 +5, 1021 +6, 1013 +7, 989 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 2227 +1, 2241 +2, 2227 +3, 2209 +4, 2239 +5, 2205 +6, 2245 +7, 2223 +0, 9851 +1, 10051 +2, 10051 +3, 9951 +4, 10091 +5, 10091 +6, 10091 +7, 9851 +0, 47013 +1, 47079 +2, 47037 +3, 46993 +4, 47039 +5, 47075 +6, 47021 +7, 47061 +0, 48087 +1, 48089 +2, 48099 +3, 48061 +4, 48085 +5, 48099 +6, 48089 +7, 48099 +0, 48645 +1, 48647 +2, 48657 +3, 48619 +4, 48643 +5, 48657 +6, 48647 +7, 48657 +0, 48805 +1, 48807 +2, 48817 +3, 48779 +4, 48803 +5, 48817 +6, 48807 +7, 48817 +0, 49509 +1, 49511 +2, 49513 +3, 49483 +4, 49483 +5, 49513 +6, 49467 +7, 49513 +0, 58785 +1, 58705 +2, 58623 +3, 58719 +4, 58819 +5, 58805 +6, 58783 +7, 58803 +0, 86183 +1, 86191 +2, 86257 +3, 86203 +4, 86225 +5, 86199 +6, 86211 +7, 86253 +0, 86669 +1, 86671 +2, 86687 +3, 86683 +4, 86683 +5, 86687 +6, 86667 +7, 86687 +0, 86669 +1, 86671 +2, 86687 +3, 86683 +4, 86683 +5, 86687 +6, 86667 +7, 86687 +0, 87007 +1, 86955 +2, 87013 +3, 87025 +4, 86981 +5, 86971 +6, 86935 +7, 86975 +0, 87953 +1, 87959 +2, 87983 +3, 87931 +4, 87953 +5, 87937 +6, 87959 +7, 87991 +0, 104475 +1, 104467 +2, 104513 +3, 103749 +4, 105069 +5, 105053 +6, 104893 +7, 105233 +0, 117837 +1, 117825 +2, 117855 +3, 117831 +4, 117867 +5, 117855 +6, 117815 +7, 117855 +0, 117837 +1, 117825 +2, 117855 +3, 117831 +4, 117867 +5, 117855 +6, 117815 +7, 117855 +0, 117837 +1, 117825 +2, 117855 +3, 117831 +4, 117867 +5, 117855 +6, 117815 +7, 117855 +0, 117837 +1, 117825 +2, 117855 +3, 117831 +4, 117867 +5, 117855 +6, 117815 +7, 117855 +0, 115479 +1, 115467 +2, 115497 +3, 115473 +4, 115509 +5, 115497 +6, 115457 +7, 115497 +0, 114251 +1, 114433 +2, 114201 +3, 114439 +4, 114601 +5, 114463 +6, 114097 +7, 114205 +0, 1 +1, 4 +2, 4 +3, 132 +4, 114217 +5, 644 +6, 4 +7, 4 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..3412380805cd41d8bb93f03f626962002d0d124e --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=256 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e254687e7228b9c7f8d0bc34555317062ebba9e4 --- /dev/null +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,26017 @@ +[15:47:17] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:47:17] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 15:47:18.384000 137212 torch/distributed/run.py:766] +W0826 15:47:18.384000 137212 torch/distributed/run.py:766] ***************************************** +W0826 15:47:18.384000 137212 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:47:18.384000 137212 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank [Gloo] Rank [Gloo] Rank 021 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 + + +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 43 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 15:47:36.705658171 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:47:36.705736543 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:47:36.707659766 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:47:36.707818446 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:47:36.709580999 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:47:36.717058978 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:47:36.718845545 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 15:47:36.873796426 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.481 seconds +time to initialize megatron (seconds): 6.482 +[after megatron is initialized] datetime: 2026-08-26 15:47:42 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8372566016 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.pre_mlp_layernorm.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.k_layernorm.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.q_layernorm.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.16.self_attention.linear_proj.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.k_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:47:47 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:47:47 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (4949.50, 4962.94) + train/valid/test-data-iterators-setup ..........: (62.91, 84.87) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 15:47:47 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32516 expected_overflow=0 exact_recv_per_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32516 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32371 expected_overflow=0 exact_recv_per_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32371 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32548 expected_overflow=0 exact_recv_per_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32548 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32629 expected_overflow=0 exact_recv_per_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32629 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33080 expected_overflow=0 exact_recv_per_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32360 expected_overflow=0 exact_recv_per_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33080 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32360 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32524 expected_overflow=0 exact_recv_per_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32524 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33037 expected_overflow=0 exact_recv_per_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=33037 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32459 expected_overflow=0 exact_recv_per_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32459 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32327 expected_overflow=0 exact_recv_per_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32327 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32488 expected_overflow=0 exact_recv_per_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32488 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33087 expected_overflow=0 exact_recv_per_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=40960 expected_recv=33087 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33002 expected_overflow=0 exact_recv_per_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=40960 expected_recv=33002 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32571 expected_overflow=0 exact_recv_per_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32571 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32957 expected_overflow=0 exact_recv_per_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32957 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33030 expected_overflow=0 exact_recv_per_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=40960 expected_recv=33030 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33000 expected_overflow=0 exact_recv_per_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33000 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33055 expected_overflow=0 exact_recv_per_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=40960 expected_recv=33055 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32973 expected_overflow=0 exact_recv_per_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32379 expected_overflow=0 exact_recv_per_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=40960 expected_recv=32973 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32379 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=40960 expected_recv=33039 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33039 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32973 expected_overflow=0 exact_recv_per_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32987 expected_overflow=0 exact_recv_per_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=40960 expected_recv=32973 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32987 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33084 expected_overflow=0 exact_recv_per_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=27 calls=1 budget=40960 expected_recv=33084 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=23 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33043 expected_overflow=0 exact_recv_per_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32421 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=24 calls=1 budget=40960 expected_recv=33043 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=28 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=24 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32938 expected_overflow=0 exact_recv_per_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32938 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=25 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32619 expected_overflow=0 exact_recv_per_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=26 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32619 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=27 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32424 expected_overflow=0 exact_recv_per_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=32 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32910 expected_overflow=0 exact_recv_per_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=28 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32424 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32910 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=33 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33064 expected_overflow=0 exact_recv_per_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=29 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=30 calls=1 budget=40960 expected_recv=33064 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=30 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32998 expected_overflow=0 exact_recv_per_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32998 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=31 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=35 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32948 expected_overflow=0 exact_recv_per_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=32 calls=1 budget=40960 expected_recv=32948 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32484 expected_overflow=0 exact_recv_per_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32484 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=33 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=34 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32510 expected_overflow=0 exact_recv_per_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32510 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=35 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33090 expected_overflow=0 exact_recv_per_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=40 calls=1 budget=40960 expected_recv=33090 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=36 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33032 expected_overflow=0 exact_recv_per_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=37 calls=1 budget=40960 expected_recv=33032 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=37 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32506 expected_overflow=0 exact_recv_per_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32413 expected_overflow=0 exact_recv_per_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32506 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32413 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=38 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=39 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32989 expected_overflow=0 exact_recv_per_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32989 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32989 expected_overflow=0 exact_recv_per_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32331 expected_overflow=0 exact_recv_per_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=40 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32989 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32331 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32424 expected_overflow=0 exact_recv_per_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32424 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=41 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33033 expected_overflow=0 exact_recv_per_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=42 calls=1 budget=40960 expected_recv=33033 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=42 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33047 expected_overflow=0 exact_recv_per_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33047 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=43 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32609, 32649, 32930, 33112, 32629, 32747, 32669] avg=32768.0000 max=33112 max_rank=4 min=32609 min_rank=1 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=44 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] avg=32768.0000 max=33096 max_rank=4 min=32625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32609, 32649, 32930, 33112, 32629, 32747, 32669] avg=32768.0000 max=33112 max_rank=4 min=32609 min_rank=1 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32352 expected_overflow=0 exact_recv_per_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=45 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=46 calls=1 budget=40960 expected_recv=32352 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] avg=32768.0000 max=33096 max_rank=4 min=32625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32609, 32649, 32930, 33112, 32629, 32747, 32669] avg=32768.0000 max=33112 max_rank=4 min=32609 min_rank=1 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32609, 32649, 32930, 33112, 32629, 32747, 32669] avg=32768.0000 max=33112 max_rank=4 min=32609 min_rank=1 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] avg=32768.0000 max=33096 max_rank=4 min=32625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=46 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33106, 32350, 32624] avg=32768.0000 max=33106 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] avg=32768.0000 max=33096 max_rank=4 min=32625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32609, 32649, 32930, 33112, 32629, 32747, 32669] avg=32768.0000 max=33112 max_rank=4 min=32609 min_rank=1 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] avg=32768.0000 max=33096 max_rank=5 min=32360 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000302 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] avg=32768.0000 max=33096 max_rank=4 min=32625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=47 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33106, 32350, 32624] avg=32768.0000 max=33106 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32467, 32759, 32548, 32627, 32872, 32898, 32723, 33250] avg=32768.0000 max=33250 max_rank=7 min=32467 min_rank=0 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] avg=32768.0000 max=33096 max_rank=5 min=32360 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000302 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] avg=32768.0000 max=33096 max_rank=7 min=32548 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33106, 32350, 32624] avg=32768.0000 max=33106 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33106, 32350, 32624] avg=32768.0000 max=33106 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32467, 32759, 32548, 32627, 32872, 32898, 32723, 33250] avg=32768.0000 max=33250 max_rank=7 min=32467 min_rank=0 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] avg=32768.0000 max=33096 max_rank=5 min=32360 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000302 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32609, 32649, 32930, 33112, 32629, 32747, 32669] avg=32768.0000 max=33112 max_rank=4 min=32609 min_rank=1 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] avg=32768.0000 max=33096 max_rank=5 min=32360 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000302 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] avg=32768.0000 max=33096 max_rank=7 min=32548 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[32516, 32951, 32980, 32525, 32972, 32782, 32439, 32979] avg=32768.0000 max=32980 max_rank=2 min=32439 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33106, 32350, 32624] avg=32768.0000 max=33106 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] avg=32768.0000 max=33096 max_rank=4 min=32625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32467, 32759, 32548, 32627, 32872, 32898, 32723, 33250] avg=32768.0000 max=33250 max_rank=7 min=32467 min_rank=0 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32609, 32649, 32930, 33112, 32629, 32747, 32669] avg=32768.0000 max=33112 max_rank=4 min=32609 min_rank=1 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] avg=32768.0000 max=33096 max_rank=5 min=32360 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000302 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32467, 32759, 32548, 32627, 32872, 32898, 32723, 33250] avg=32768.0000 max=33250 max_rank=7 min=32467 min_rank=0 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33113, 32889, 32557, 32784, 32772] avg=32768.0000 max=33113 max_rank=3 min=32557 min_rank=5 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] avg=32768.0000 max=33096 max_rank=7 min=32548 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] avg=32768.0000 max=33096 max_rank=4 min=32625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32609, 32649, 32930, 33112, 32629, 32747, 32669] avg=32768.0000 max=33112 max_rank=4 min=32609 min_rank=1 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] avg=32768.0000 max=33096 max_rank=7 min=32548 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] avg=32768.0000 max=33096 max_rank=3 min=32574 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32467, 32759, 32548, 32627, 32872, 32898, 32723, 33250] avg=32768.0000 max=33250 max_rank=7 min=32467 min_rank=0 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32799, 32625, 32649, 32930, 33096, 32629, 32747, 32669] avg=32768.0000 max=33096 max_rank=4 min=32625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] avg=32768.0000 max=33096 max_rank=7 min=32548 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33107, 32823, 32316, 32784, 32782] avg=32768.0000 max=33107 max_rank=3 min=32316 min_rank=5 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33113, 32889, 32557, 32784, 32772] avg=32768.0000 max=33113 max_rank=3 min=32557 min_rank=5 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33106, 32350, 32624] avg=32768.0000 max=33106 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] avg=32768.0000 max=33096 max_rank=3 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] avg=32768.0000 max=33096 max_rank=3 min=32574 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[32852, 32371, 32985, 32855, 33080, 32660, 32636, 32705] avg=32768.0000 max=33080 max_rank=4 min=32371 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] avg=32768.0000 max=33096 max_rank=5 min=32360 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000302 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33106, 32350, 32624] avg=32768.0000 max=33106 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33113, 32889, 32557, 32784, 32772] avg=32768.0000 max=33113 max_rank=3 min=32557 min_rank=5 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33113, 32889, 32557, 32784, 32772] avg=32768.0000 max=33113 max_rank=3 min=32557 min_rank=5 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33107, 32823, 32316, 32784, 32782] avg=32768.0000 max=33107 max_rank=3 min=32316 min_rank=5 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] avg=32768.0000 max=33096 max_rank=5 min=32360 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000302 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] avg=32768.0000 max=33096 max_rank=3 min=32574 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33106, 32350, 32624] avg=32768.0000 max=33106 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010315 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32467, 32759, 32548, 32627, 32872, 32898, 32723, 33250] avg=32768.0000 max=33250 max_rank=7 min=32467 min_rank=0 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] avg=32768.0000 max=33096 max_rank=3 min=32574 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] avg=32768.0000 max=33096 max_rank=3 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32567, 32933, 32719, 32752, 33093, 33096, 32360, 32624] avg=32768.0000 max=33096 max_rank=5 min=32360 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=10 max_load_reduction_pct=0.000302 moved_to_replicas=10 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] avg=32768.0000 max=33096 max_rank=7 min=32548 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[32467, 32759, 32548, 32627, 32872, 32898, 32723, 33250] avg=32768.0000 max=33250 max_rank=7 min=32467 min_rank=0 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33107, 32823, 32316, 32784, 32782] avg=32768.0000 max=33107 max_rank=3 min=32316 min_rank=5 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32388, 33250] avg=32768.0000 max=33250 max_rank=7 min=32388 min_rank=6 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33113, 32889, 32557, 32784, 32772] avg=32768.0000 max=33113 max_rank=3 min=32557 min_rank=5 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] avg=32768.0000 max=33096 max_rank=7 min=32548 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33107, 32823, 32316, 32784, 32782] avg=32768.0000 max=33107 max_rank=3 min=32316 min_rank=5 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] avg=32768.0000 max=33096 max_rank=3 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32467, 32759, 32548, 32627, 32872, 32898, 32723, 33250] avg=32768.0000 max=33250 max_rank=7 min=32467 min_rank=0 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] avg=32768.0000 max=33096 max_rank=7 min=32542 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] avg=32768.0000 max=33096 max_rank=3 min=32574 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] avg=32768.0000 max=33096 max_rank=3 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32621, 32759, 32548, 32627, 32872, 32898, 32723, 33096] avg=32768.0000 max=33096 max_rank=7 min=32548 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33107, 32823, 32316, 32784, 32782] avg=32768.0000 max=33107 max_rank=3 min=32316 min_rank=5 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32388, 33250] avg=32768.0000 max=33250 max_rank=7 min=32388 min_rank=6 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33113, 32889, 32557, 32784, 32772] avg=32768.0000 max=33113 max_rank=3 min=32557 min_rank=5 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] avg=32768.0000 max=33096 max_rank=3 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] avg=32768.0000 max=33096 max_rank=7 min=32542 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] avg=32768.0000 max=33096 max_rank=3 min=32574 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32905, 32843, 32753, 32713, 32767, 32518, 33037, 32608] avg=32768.0000 max=33037 max_rank=6 min=32518 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33113, 32889, 32557, 32784, 32772] avg=32768.0000 max=33113 max_rank=3 min=32557 min_rank=5 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32388, 33250] avg=32768.0000 max=33250 max_rank=7 min=32388 min_rank=6 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] avg=32768.0000 max=33096 max_rank=3 min=32574 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] avg=32768.0000 max=33096 max_rank=7 min=32542 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32388, 33250] avg=32768.0000 max=33250 max_rank=7 min=32388 min_rank=6 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33107, 32823, 32316, 32784, 32782] avg=32768.0000 max=33107 max_rank=3 min=32316 min_rank=5 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33113, 32889, 32557, 32784, 32772] avg=32768.0000 max=33113 max_rank=3 min=32557 min_rank=5 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] avg=32768.0000 max=33096 max_rank=7 min=32542 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] avg=32768.0000 max=33096 max_rank=3 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32780, 32596, 32653, 33096, 32889, 32574, 32784, 32772] avg=32768.0000 max=33096 max_rank=3 min=32574 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33107, 32823, 32316, 32784, 32782] avg=32768.0000 max=33107 max_rank=3 min=32316 min_rank=5 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32388, 33250] avg=32768.0000 max=33250 max_rank=7 min=32388 min_rank=6 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] avg=32768.0000 max=33096 max_rank=3 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33107, 32823, 32316, 32784, 32782] avg=32768.0000 max=33107 max_rank=3 min=32316 min_rank=5 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] avg=32768.0000 max=33096 max_rank=7 min=32542 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[32638, 32928, 32766, 33096, 32823, 32327, 32784, 32782] avg=32768.0000 max=33096 max_rank=3 min=32327 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32388, 33250] avg=32768.0000 max=33250 max_rank=7 min=32388 min_rank=6 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] avg=32768.0000 max=33096 max_rank=7 min=32542 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32388, 33250] avg=32768.0000 max=33250 max_rank=7 min=32388 min_rank=6 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32524, 32971, 32927, 32917, 32858, 32686, 32488, 32773] avg=32768.0000 max=32971 max_rank=1 min=32488 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[32384, 32571, 32661, 32852, 32814, 33138, 32886, 32838] avg=32768.0000 max=33138 max_rank=5 min=32384 min_rank=0 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] avg=32768.0000 max=33096 max_rank=7 min=32542 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32388, 33250] avg=32768.0000 max=33250 max_rank=7 min=32388 min_rank=6 imbalance_ratio=1.014709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32953, 32747, 32885, 32665, 32589, 32667, 32542, 33096] avg=32768.0000 max=33096 max_rank=7 min=32542 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=154 max_load_reduction_pct=0.004632 moved_to_replicas=154 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[32384, 32571, 32661, 32852, 32814, 33138, 32886, 32838] avg=32768.0000 max=33138 max_rank=5 min=32384 min_rank=0 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[32822, 32459, 32749, 32943, 32788, 32957, 32696, 32730] avg=32768.0000 max=32957 max_rank=5 min=32459 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[32384, 32571, 32661, 32852, 32814, 33138, 32886, 32838] avg=32768.0000 max=33138 max_rank=5 min=32384 min_rank=0 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[32384, 32571, 32661, 32852, 32814, 33138, 32886, 32838] avg=32768.0000 max=33138 max_rank=5 min=32384 min_rank=0 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[32837, 32604, 32766, 32831, 32759, 32631, 32856, 32860] avg=32768.0000 max=32860 max_rank=7 min=32604 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33118, 32529, 32713, 32674] avg=32768.0000 max=33118 max_rank=4 min=32529 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[32384, 32571, 32661, 32852, 32814, 33138, 32886, 32838] avg=32768.0000 max=33138 max_rank=5 min=32384 min_rank=0 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] avg=32768.0000 max=33096 max_rank=4 min=32551 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[32650, 33087, 32609, 32785, 32743, 32526, 32851, 32893] avg=32768.0000 max=33087 max_rank=1 min=32526 min_rank=5 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[32384, 32571, 32661, 32852, 32814, 33138, 32886, 32838] avg=32768.0000 max=33138 max_rank=5 min=32384 min_rank=0 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33118, 32529, 32713, 32674] avg=32768.0000 max=33118 max_rank=4 min=32529 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[32384, 32571, 32661, 32852, 32814, 33138, 32886, 32838] avg=32768.0000 max=33138 max_rank=5 min=32384 min_rank=0 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] avg=32768.0000 max=33096 max_rank=4 min=32551 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[33002, 32773, 32641, 32794, 32973, 32591, 32838, 32532] avg=32768.0000 max=33002 max_rank=0 min=32532 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33118, 32529, 32713, 32674] avg=32768.0000 max=33118 max_rank=4 min=32529 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[32384, 32571, 32661, 32852, 32814, 33138, 32886, 32838] avg=32768.0000 max=33138 max_rank=5 min=32384 min_rank=0 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] avg=32768.0000 max=33096 max_rank=4 min=32551 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[32426, 32571, 32661, 32852, 32814, 33096, 32886, 32838] avg=32768.0000 max=33096 max_rank=5 min=32426 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33118, 32529, 32713, 32674] avg=32768.0000 max=33118 max_rank=4 min=32529 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] avg=32768.0000 max=33096 max_rank=4 min=32551 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33118, 32529, 32713, 32674] avg=32768.0000 max=33118 max_rank=4 min=32529 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[32846, 32829, 32573, 33030, 32718, 32494, 32854, 32800] avg=32768.0000 max=33030 max_rank=3 min=32494 min_rank=5 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] avg=32768.0000 max=33096 max_rank=4 min=32551 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33118, 32529, 32713, 32674] avg=32768.0000 max=33118 max_rank=4 min=32529 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33118, 32529, 32713, 32674] avg=32768.0000 max=33118 max_rank=4 min=32529 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] avg=32768.0000 max=33096 max_rank=4 min=32551 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[32599, 33055, 33000, 32609, 32732, 32839, 32889, 32421] avg=32768.0000 max=33055 max_rank=1 min=32421 min_rank=7 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] avg=32768.0000 max=33096 max_rank=4 min=32551 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33118, 32529, 32713, 32674] avg=32768.0000 max=33118 max_rank=4 min=32529 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[32750, 32744, 32863, 32753, 33096, 32551, 32713, 32674] avg=32768.0000 max=33096 max_rank=4 min=32551 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[32379, 32816, 32708, 33039, 32867, 32721, 32658, 32956] avg=32768.0000 max=33039 max_rank=3 min=32379 min_rank=0 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[32683, 32790, 32544, 32874, 32805, 32636, 32900, 32912] avg=32768.0000 max=32912 max_rank=7 min=32544 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=26 tokens_per_ep_rank=[32469, 32653, 33103, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33103 max_rank=2 min=32469 min_rank=0 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=26 tokens_per_ep_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[32673, 32715, 32881, 32782, 32722, 32624, 32868, 32879] avg=32768.0000 max=32881 max_rank=2 min=32624 min_rank=5 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=26 tokens_per_ep_rank=[32469, 32653, 33103, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33103 max_rank=2 min=32469 min_rank=0 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[32848, 32826, 32925, 32778, 32518, 32719, 32565, 32965] avg=32768.0000 max=32965 max_rank=7 min=32518 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=26 tokens_per_ep_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=26 tokens_per_ep_rank=[32469, 32653, 33103, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33103 max_rank=2 min=32469 min_rank=0 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[32738, 32567, 32770, 32856, 32973, 32913, 32582, 32745] avg=32768.0000 max=32973 max_rank=4 min=32567 min_rank=1 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=26 tokens_per_ep_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=26 tokens_per_ep_rank=[32469, 32653, 33103, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33103 max_rank=2 min=32469 min_rank=0 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[32904, 32804, 32719, 32765, 32772, 32569, 32838, 32773] avg=32768.0000 max=32904 max_rank=0 min=32569 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=26 tokens_per_ep_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=26 tokens_per_ep_rank=[32469, 32653, 33103, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33103 max_rank=2 min=32469 min_rank=0 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=26 tokens_per_ep_rank=[32469, 32653, 33103, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33103 max_rank=2 min=32469 min_rank=0 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=26 tokens_per_ep_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=26 tokens_per_ep_rank=[32469, 32653, 33103, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33103 max_rank=2 min=32469 min_rank=0 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=26 tokens_per_ep_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=26 tokens_per_ep_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=25 tokens_per_ep_rank=[32458, 32639, 32753, 32669, 32638, 33043, 32905, 33039] avg=32768.0000 max=33043 max_rank=5 min=32458 min_rank=0 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=26 tokens_per_ep_rank=[32469, 32653, 33103, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33103 max_rank=2 min=32469 min_rank=0 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=32 tokens_per_ep_rank=[33115, 32682, 32619, 32446, 32799, 32784, 32868, 32831] avg=32768.0000 max=33115 max_rank=0 min=32446 min_rank=3 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=26 tokens_per_ep_rank=[32476, 32653, 33096, 32784, 32900, 32675, 32635, 32925] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=32 tokens_per_ep_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=27 tokens_per_ep_rank=[32970, 32776, 32705, 32522, 32898, 32704, 32901, 32668] avg=32768.0000 max=32970 max_rank=0 min=32522 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=32 tokens_per_ep_rank=[33115, 32682, 32619, 32446, 32799, 32784, 32868, 32831] avg=32768.0000 max=33115 max_rank=0 min=32446 min_rank=3 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=28 tokens_per_ep_rank=[33084, 32528, 32987, 32906, 32838, 32684, 32639, 32478] avg=32768.0000 max=33084 max_rank=0 min=32478 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=32 tokens_per_ep_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=32 tokens_per_ep_rank=[33115, 32682, 32619, 32446, 32799, 32784, 32868, 32831] avg=32768.0000 max=33115 max_rank=0 min=32446 min_rank=3 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=29 tokens_per_ep_rank=[32421, 32937, 32862, 33095, 32664, 32649, 32801, 32715] avg=32768.0000 max=33095 max_rank=3 min=32421 min_rank=0 imbalance_ratio=1.009979 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=32 tokens_per_ep_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=32 tokens_per_ep_rank=[33115, 32682, 32619, 32446, 32799, 32784, 32868, 32831] avg=32768.0000 max=33115 max_rank=0 min=32446 min_rank=3 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=30 tokens_per_ep_rank=[32764, 32938, 32711, 32931, 32509, 32470, 32910, 32911] avg=32768.0000 max=32938 max_rank=1 min=32470 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=32 tokens_per_ep_rank=[33115, 32682, 32619, 32446, 32799, 32784, 32868, 32831] avg=32768.0000 max=33115 max_rank=0 min=32446 min_rank=3 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=32 tokens_per_ep_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=32 tokens_per_ep_rank=[33115, 32682, 32619, 32446, 32799, 32784, 32868, 32831] avg=32768.0000 max=33115 max_rank=0 min=32446 min_rank=3 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=32 tokens_per_ep_rank=[33115, 32682, 32619, 32446, 32799, 32784, 32868, 32831] avg=32768.0000 max=33115 max_rank=0 min=32446 min_rank=3 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=32 tokens_per_ep_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=32 tokens_per_ep_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=31 tokens_per_ep_rank=[32551, 32758, 32869, 32627, 32850, 32862, 33064, 32563] avg=32768.0000 max=33064 max_rank=6 min=32551 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=32 tokens_per_ep_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=32 tokens_per_ep_rank=[33115, 32682, 32619, 32446, 32799, 32784, 32868, 32831] avg=32768.0000 max=33115 max_rank=0 min=32446 min_rank=3 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=32 tokens_per_ep_rank=[33096, 32682, 32619, 32465, 32799, 32784, 32868, 32831] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=39 tokens_per_ep_rank=[32480, 32750, 32656, 32931, 32892, 32506, 32803, 33126] avg=32768.0000 max=33126 max_rank=7 min=32480 min_rank=0 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=33 tokens_per_ep_rank=[33016, 32773, 32853, 32424, 32948, 32942, 32505, 32683] avg=32768.0000 max=33016 max_rank=0 min=32424 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=39 tokens_per_ep_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] avg=32768.0000 max=33096 max_rank=7 min=32506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=30 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=34 tokens_per_ep_rank=[33005, 32652, 32825, 32993, 32724, 32677, 32484, 32784] avg=32768.0000 max=33005 max_rank=0 min=32484 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=39 tokens_per_ep_rank=[32480, 32750, 32656, 32931, 32892, 32506, 32803, 33126] avg=32768.0000 max=33126 max_rank=7 min=32480 min_rank=0 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=35 tokens_per_ep_rank=[32615, 32777, 32866, 32740, 32880, 32907, 32713, 32646] avg=32768.0000 max=32907 max_rank=5 min=32615 min_rank=0 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=39 tokens_per_ep_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] avg=32768.0000 max=33096 max_rank=7 min=32506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=30 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=36 tokens_per_ep_rank=[32716, 32708, 32769, 33010, 32725, 32556, 32866, 32794] avg=32768.0000 max=33010 max_rank=3 min=32556 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=39 tokens_per_ep_rank=[32480, 32750, 32656, 32931, 32892, 32506, 32803, 33126] avg=32768.0000 max=33126 max_rank=7 min=32480 min_rank=0 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=39 tokens_per_ep_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] avg=32768.0000 max=33096 max_rank=7 min=32506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=30 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=37 tokens_per_ep_rank=[32606, 32826, 32762, 32723, 32851, 32486, 32892, 32998] avg=32768.0000 max=32998 max_rank=7 min=32486 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=39 tokens_per_ep_rank=[32480, 32750, 32656, 32931, 32892, 32506, 32803, 33126] avg=32768.0000 max=33126 max_rank=7 min=32480 min_rank=0 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=39 tokens_per_ep_rank=[32480, 32750, 32656, 32931, 32892, 32506, 32803, 33126] avg=32768.0000 max=33126 max_rank=7 min=32480 min_rank=0 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=39 tokens_per_ep_rank=[32480, 32750, 32656, 32931, 32892, 32506, 32803, 33126] avg=32768.0000 max=33126 max_rank=7 min=32480 min_rank=0 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=39 tokens_per_ep_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] avg=32768.0000 max=33096 max_rank=7 min=32506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=30 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=39 tokens_per_ep_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] avg=32768.0000 max=33096 max_rank=7 min=32506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=30 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=39 tokens_per_ep_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] avg=32768.0000 max=33096 max_rank=7 min=32506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=30 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=39 tokens_per_ep_rank=[32480, 32750, 32656, 32931, 32892, 32506, 32803, 33126] avg=32768.0000 max=33126 max_rank=7 min=32480 min_rank=0 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=38 tokens_per_ep_rank=[32518, 32574, 32873, 32833, 32793, 33032, 32827, 32694] avg=32768.0000 max=33032 max_rank=5 min=32518 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33174, 32668, 32677, 32834, 32725, 32637, 32719] avg=32768.0000 max=33174 max_rank=1 min=32637 min_rank=6 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=39 tokens_per_ep_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] avg=32768.0000 max=33096 max_rank=7 min=32506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=30 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] avg=32768.0000 max=33096 max_rank=1 min=32668 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=39 tokens_per_ep_rank=[32480, 32750, 32656, 32931, 32892, 32506, 32803, 33126] avg=32768.0000 max=33126 max_rank=7 min=32480 min_rank=0 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=39 tokens_per_ep_rank=[32510, 32750, 32656, 32931, 32892, 32506, 32803, 33096] avg=32768.0000 max=33096 max_rank=7 min=32506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=30 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33174, 32668, 32677, 32834, 32725, 32637, 32719] avg=32768.0000 max=33174 max_rank=1 min=32637 min_rank=6 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=40 tokens_per_ep_rank=[32887, 32691, 32901, 32782, 32592, 32849, 32646, 32796] avg=32768.0000 max=32901 max_rank=2 min=32592 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] avg=32768.0000 max=33096 max_rank=1 min=32668 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33159, 32289, 32424] avg=32768.0000 max=33159 max_rank=5 min=32289 min_rank=6 imbalance_ratio=1.011932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=41 tokens_per_ep_rank=[32698, 32620, 33090, 32583, 32790, 32835, 32726, 32802] avg=32768.0000 max=33090 max_rank=2 min=32583 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33174, 32668, 32677, 32834, 32725, 32637, 32719] avg=32768.0000 max=33174 max_rank=1 min=32637 min_rank=6 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] avg=32768.0000 max=33096 max_rank=5 min=32352 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63 max_load_reduction_pct=0.001900 moved_to_replicas=63 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] avg=32768.0000 max=33096 max_rank=1 min=32668 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=42 tokens_per_ep_rank=[32652, 32795, 32662, 32809, 32971, 32838, 32613, 32804] avg=32768.0000 max=32971 max_rank=4 min=32613 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33174, 32668, 32677, 32834, 32725, 32637, 32719] avg=32768.0000 max=33174 max_rank=1 min=32637 min_rank=6 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33174, 32668, 32677, 32834, 32725, 32637, 32719] avg=32768.0000 max=33174 max_rank=1 min=32637 min_rank=6 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33174, 32668, 32677, 32834, 32725, 32637, 32719] avg=32768.0000 max=33174 max_rank=1 min=32637 min_rank=6 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] avg=32768.0000 max=33096 max_rank=1 min=32668 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] avg=32768.0000 max=33096 max_rank=1 min=32668 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33174, 32668, 32677, 32834, 32725, 32637, 32719] avg=32768.0000 max=33174 max_rank=1 min=32637 min_rank=6 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] avg=32768.0000 max=33096 max_rank=1 min=32668 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=43 tokens_per_ep_rank=[32837, 32413, 32731, 32963, 32678, 33033, 32818, 32671] avg=32768.0000 max=33033 max_rank=5 min=32413 min_rank=1 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33159, 32289, 32424] avg=32768.0000 max=33159 max_rank=5 min=32289 min_rank=6 imbalance_ratio=1.011932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] avg=32768.0000 max=33096 max_rank=1 min=32668 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] avg=32768.0000 max=33096 max_rank=5 min=32352 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63 max_load_reduction_pct=0.001900 moved_to_replicas=63 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33174, 32668, 32677, 32834, 32725, 32637, 32719] avg=32768.0000 max=33174 max_rank=1 min=32637 min_rank=6 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=44 tokens_per_ep_rank=[32710, 33096, 32668, 32677, 32834, 32725, 32715, 32719] avg=32768.0000 max=33096 max_rank=1 min=32668 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33159, 32289, 32424] avg=32768.0000 max=33159 max_rank=5 min=32289 min_rank=6 imbalance_ratio=1.011932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] avg=32768.0000 max=33096 max_rank=5 min=32352 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63 max_load_reduction_pct=0.001900 moved_to_replicas=63 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=45 tokens_per_ep_rank=[32331, 32989, 32782, 32901, 32760, 32912, 32820, 32649] avg=32768.0000 max=32989 max_rank=1 min=32331 min_rank=0 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33159, 32289, 32424] avg=32768.0000 max=33159 max_rank=5 min=32289 min_rank=6 imbalance_ratio=1.011932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33159, 32289, 32424] avg=32768.0000 max=33159 max_rank=5 min=32289 min_rank=6 imbalance_ratio=1.011932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33159, 32289, 32424] avg=32768.0000 max=33159 max_rank=5 min=32289 min_rank=6 imbalance_ratio=1.011932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] avg=32768.0000 max=33096 max_rank=5 min=32352 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63 max_load_reduction_pct=0.001900 moved_to_replicas=63 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] avg=32768.0000 max=33096 max_rank=5 min=32352 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63 max_load_reduction_pct=0.001900 moved_to_replicas=63 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33159, 32289, 32424] avg=32768.0000 max=33159 max_rank=5 min=32289 min_rank=6 imbalance_ratio=1.011932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] avg=32768.0000 max=33096 max_rank=5 min=32352 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63 max_load_reduction_pct=0.001900 moved_to_replicas=63 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=46 tokens_per_ep_rank=[32805, 32661, 32878, 32679, 32645, 32582, 32905, 32989] avg=32768.0000 max=32989 max_rank=7 min=32582 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] avg=32768.0000 max=33096 max_rank=5 min=32352 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63 max_load_reduction_pct=0.001900 moved_to_replicas=63 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33159, 32289, 32424] avg=32768.0000 max=33159 max_rank=5 min=32289 min_rank=6 imbalance_ratio=1.011932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=47 tokens_per_ep_rank=[33047, 32782, 32724, 32718, 33001, 33096, 32352, 32424] avg=32768.0000 max=33096 max_rank=5 min=32352 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=63 max_load_reduction_pct=0.001900 moved_to_replicas=63 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=48 tokens_per_ep_rank=[32817, 32917, 32738, 32713, 32634, 32600, 32921, 32804] avg=32768.0000 max=32921 max_rank=6 min=32600 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=48 total_expected_recv=1573565 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343936 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=48 total_expected_recv=1573654 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343424 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=48 total_expected_recv=1572299 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345088 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=48 total_expected_recv=1571151 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=48 total_expected_recv=1571863 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346368 peak_demand=33064 required_capacity=1.009033 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=48 total_expected_recv=1575133 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342784 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=48 total_expected_recv=1574345 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342912 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=48 total_expected_recv=1570902 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346496 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:48:14] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 26975.3 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095307E+01 | loss scale: 1.0 | grad norm: 54.372 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 72349.68896484375 | max allocated: 79886.0634765625 | reserved: 80310.0 | max reserved: 80310.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32997 expected_overflow=0 exact_recv_per_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32997 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32369 expected_overflow=0 exact_recv_per_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32369 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32488 expected_overflow=0 exact_recv_per_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32488 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32507 expected_overflow=0 exact_recv_per_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32507 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32947 expected_overflow=0 exact_recv_per_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32947 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33026 expected_overflow=0 exact_recv_per_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=33026 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32974 expected_overflow=0 exact_recv_per_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32974 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32380 expected_overflow=0 exact_recv_per_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=40960 expected_recv=32380 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33086 expected_overflow=0 exact_recv_per_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=40960 expected_recv=33086 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33060 expected_overflow=0 exact_recv_per_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=40960 expected_recv=33060 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33060 expected_overflow=0 exact_recv_per_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33060 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32515 expected_overflow=0 exact_recv_per_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33032 expected_overflow=0 exact_recv_per_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32515 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33032 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33008 expected_overflow=0 exact_recv_per_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=40960 expected_recv=33008 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33068 expected_overflow=0 exact_recv_per_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=40960 expected_recv=33068 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32998 expected_overflow=0 exact_recv_per_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32998 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33004 expected_overflow=0 exact_recv_per_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33004 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32950 expected_overflow=0 exact_recv_per_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32950 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32947 expected_overflow=0 exact_recv_per_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32947 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32989 expected_overflow=0 exact_recv_per_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32534 expected_overflow=0 exact_recv_per_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=23 calls=1 budget=40960 expected_recv=32989 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32534 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=23 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=25 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=26 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=24 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32938 expected_overflow=0 exact_recv_per_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32938 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=25 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32936 expected_overflow=0 exact_recv_per_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32557 expected_overflow=0 exact_recv_per_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32936 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=26 calls=1 budget=40960 expected_recv=32557 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=28 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32397 expected_overflow=0 exact_recv_per_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32397 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=27 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33020 expected_overflow=0 exact_recv_per_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=30 calls=1 budget=40960 expected_recv=33020 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32554 expected_overflow=0 exact_recv_per_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32554 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=28 calls=1 budget=40960 expected_recv=32536 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32949 expected_overflow=0 exact_recv_per_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32949 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32960 expected_overflow=0 exact_recv_per_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32960 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=29 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32915 expected_overflow=0 exact_recv_per_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32915 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33027 expected_overflow=0 exact_recv_per_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32595 expected_overflow=0 exact_recv_per_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33027 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32595 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32470 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32344 expected_overflow=0 exact_recv_per_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=30 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32344 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=31 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33052 expected_overflow=0 exact_recv_per_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=32 calls=1 budget=40960 expected_recv=33052 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=32 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32514 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33049 expected_overflow=0 exact_recv_per_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=33 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=34 calls=1 budget=40960 expected_recv=33049 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=34 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=35 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=36 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=36 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32601 expected_overflow=0 exact_recv_per_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=37 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32601 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=38 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=39 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32296 expected_overflow=0 exact_recv_per_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=40 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32296 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33061 expected_overflow=0 exact_recv_per_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=41 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=44 calls=1 budget=40960 expected_recv=33061 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32989 expected_overflow=0 exact_recv_per_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=42 calls=1 budget=40960 expected_recv=32989 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33000 expected_overflow=0 exact_recv_per_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=42 calls=1 budget=40960 expected_recv=33000 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32570 expected_overflow=0 exact_recv_per_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=43 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32570 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32595 expected_overflow=0 exact_recv_per_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=44 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32595 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=45 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[33170, 32650, 32886, 32655, 32672, 32603, 32628, 32880] avg=32768.0000 max=33170 max_rank=0 min=32603 min_rank=5 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] avg=32768.0000 max=33096 max_rank=0 min=32628 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=46 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[33170, 32650, 32886, 32655, 32672, 32603, 32628, 32880] avg=32768.0000 max=33170 max_rank=0 min=32603 min_rank=5 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[33170, 32650, 32886, 32655, 32672, 32603, 32628, 32880] avg=32768.0000 max=33170 max_rank=0 min=32603 min_rank=5 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] avg=32768.0000 max=33096 max_rank=0 min=32628 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] avg=32768.0000 max=33096 max_rank=0 min=32628 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[33170, 32650, 32886, 32655, 32672, 32603, 32628, 32880] avg=32768.0000 max=33170 max_rank=0 min=32603 min_rank=5 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=47 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[33170, 32650, 32886, 32655, 32672, 32603, 32628, 32880] avg=32768.0000 max=33170 max_rank=0 min=32603 min_rank=5 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] avg=32768.0000 max=33096 max_rank=0 min=32628 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] avg=32768.0000 max=33096 max_rank=0 min=32628 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[33170, 32650, 32886, 32655, 32672, 32603, 32628, 32880] avg=32768.0000 max=33170 max_rank=0 min=32603 min_rank=5 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[33170, 32650, 32886, 32655, 32672, 32603, 32628, 32880] avg=32768.0000 max=33170 max_rank=0 min=32603 min_rank=5 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] avg=32768.0000 max=33096 max_rank=0 min=32628 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32812, 32750, 32759, 32540, 32986, 32509, 32816, 32972] avg=32768.0000 max=32986 max_rank=4 min=32509 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] avg=32768.0000 max=33096 max_rank=0 min=32628 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[33170, 32650, 32886, 32655, 32672, 32603, 32628, 32880] avg=32768.0000 max=33170 max_rank=0 min=32603 min_rank=5 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33181, 32422, 32977, 32488, 32772, 32546] avg=32768.0000 max=33181 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.012604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[33096, 32650, 32886, 32655, 32672, 32677, 32628, 32880] avg=32768.0000 max=33096 max_rank=0 min=32628 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] avg=32768.0000 max=33096 max_rank=2 min=32488 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.002562 moved_to_replicas=85 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33181, 32422, 32977, 32488, 32772, 32546] avg=32768.0000 max=33181 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.012604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33181, 32422, 32977, 32488, 32772, 32546] avg=32768.0000 max=33181 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.012604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32886, 32719, 32847, 32643, 32714, 32810, 32693, 32832] avg=32768.0000 max=32886 max_rank=0 min=32643 min_rank=3 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] avg=32768.0000 max=33096 max_rank=2 min=32488 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.002562 moved_to_replicas=85 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] avg=32768.0000 max=33096 max_rank=2 min=32488 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.002562 moved_to_replicas=85 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33181, 32422, 32977, 32488, 32772, 32546] avg=32768.0000 max=33181 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.012604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33181, 32422, 32977, 32488, 32772, 32546] avg=32768.0000 max=33181 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.012604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] avg=32768.0000 max=33096 max_rank=2 min=32488 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.002562 moved_to_replicas=85 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] avg=32768.0000 max=33096 max_rank=2 min=32488 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.002562 moved_to_replicas=85 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[32885, 32496, 32935, 32369, 32600, 32997, 32861, 33001] avg=32768.0000 max=33001 max_rank=7 min=32369 min_rank=3 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33181, 32422, 32977, 32488, 32772, 32546] avg=32768.0000 max=33181 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.012604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] avg=32768.0000 max=33096 max_rank=2 min=32488 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.002562 moved_to_replicas=85 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32824, 32855, 32634, 32804, 32529, 32847, 32690, 32961] avg=32768.0000 max=32961 max_rank=7 min=32529 min_rank=4 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33181, 32422, 32977, 32488, 32772, 32546] avg=32768.0000 max=33181 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.012604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] avg=32768.0000 max=33096 max_rank=2 min=32488 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.002562 moved_to_replicas=85 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33181, 32422, 32977, 32488, 32772, 32546] avg=32768.0000 max=33181 max_rank=2 min=32422 min_rank=3 imbalance_ratio=1.012604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32853, 32905, 33096, 32507, 32977, 32488, 32772, 32546] avg=32768.0000 max=33096 max_rank=2 min=32488 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.002562 moved_to_replicas=85 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[32824, 32743, 32606, 32753, 32826, 32750, 32865, 32777] avg=32768.0000 max=32865 max_rank=6 min=32606 min_rank=2 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32984, 32762, 32833, 32751, 32547, 32798, 32771, 32698] avg=32768.0000 max=32984 max_rank=0 min=32547 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32837, 33026, 32947, 32568, 32762, 32525, 32763, 32716] avg=32768.0000 max=33026 max_rank=1 min=32525 min_rank=5 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[33016, 32743, 32572, 32590, 32833, 32671, 32954, 32765] avg=32768.0000 max=33016 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[32754, 32796, 32992, 32974, 32814, 32576, 32428, 32810] avg=32768.0000 max=32992 max_rank=2 min=32428 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[32380, 32808, 32955, 32429, 32903, 32758, 32930, 32981] avg=32768.0000 max=32981 max_rank=7 min=32380 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32548, 32752, 32726, 32691, 33166, 32925] avg=32768.0000 max=33166 max_rank=6 min=32548 min_rank=2 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] avg=32768.0000 max=33096 max_rank=6 min=32591 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[32920, 32714, 32774, 33086, 32810, 32567, 32640, 32633] avg=32768.0000 max=33086 max_rank=3 min=32567 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32548, 32752, 32726, 32691, 33166, 32925] avg=32768.0000 max=33166 max_rank=6 min=32548 min_rank=2 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32548, 32752, 32726, 32691, 33166, 32925] avg=32768.0000 max=33166 max_rank=6 min=32548 min_rank=2 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] avg=32768.0000 max=33096 max_rank=6 min=32591 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32548, 32752, 32726, 32691, 33166, 32925] avg=32768.0000 max=33166 max_rank=6 min=32548 min_rank=2 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32548, 32752, 32726, 32691, 33166, 32925] avg=32768.0000 max=33166 max_rank=6 min=32548 min_rank=2 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] avg=32768.0000 max=33096 max_rank=6 min=32591 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[33060, 32884, 32841, 32552, 32728, 32655, 32696, 32728] avg=32768.0000 max=33060 max_rank=0 min=32552 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] avg=32768.0000 max=33096 max_rank=6 min=32591 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] avg=32768.0000 max=33096 max_rank=6 min=32591 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[32810, 32760, 32621, 32972, 32443, 32684, 33034, 32820] avg=32768.0000 max=33034 max_rank=6 min=32443 min_rank=4 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32548, 32752, 32726, 32691, 33166, 32925] avg=32768.0000 max=33166 max_rank=6 min=32548 min_rank=2 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] avg=32768.0000 max=33096 max_rank=6 min=32591 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[32654, 32827, 32922, 32928, 32736, 32879, 32661, 32537] avg=32768.0000 max=32928 max_rank=3 min=32537 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32548, 32752, 32726, 32691, 33166, 32925] avg=32768.0000 max=33166 max_rank=6 min=32548 min_rank=2 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32548, 32752, 32726, 32691, 33166, 32925] avg=32768.0000 max=33166 max_rank=6 min=32548 min_rank=2 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] avg=32768.0000 max=33096 max_rank=6 min=32591 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[32591, 32745, 32618, 32752, 32726, 32691, 33096, 32925] avg=32768.0000 max=33096 max_rank=6 min=32591 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33108, 32694, 32384, 32849, 32568] avg=32768.0000 max=33108 max_rank=3 min=32384 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] avg=32768.0000 max=33096 max_rank=3 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[32661, 33032, 32839, 33023, 33060, 32456, 32485, 32588] avg=32768.0000 max=33060 max_rank=4 min=32456 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33108, 32694, 32384, 32849, 32568] avg=32768.0000 max=33108 max_rank=3 min=32384 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33108, 32694, 32384, 32849, 32568] avg=32768.0000 max=33108 max_rank=3 min=32384 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] avg=32768.0000 max=33096 max_rank=3 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33108, 32694, 32384, 32849, 32568] avg=32768.0000 max=33108 max_rank=3 min=32384 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33108, 32694, 32384, 32849, 32568] avg=32768.0000 max=33108 max_rank=3 min=32384 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[32721, 32605, 32888, 33008, 32515, 32788, 32842, 32777] avg=32768.0000 max=33008 max_rank=3 min=32515 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] avg=32768.0000 max=33096 max_rank=3 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] avg=32768.0000 max=33096 max_rank=3 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] avg=32768.0000 max=33096 max_rank=3 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[32861, 32757, 32634, 32794, 32692, 32803, 32998, 32605] avg=32768.0000 max=32998 max_rank=6 min=32605 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33108, 32694, 32384, 32849, 32568] avg=32768.0000 max=33108 max_rank=3 min=32384 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=25 tokens_per_ep_rank=[33115, 32836, 32566, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33115 max_rank=0 min=32566 min_rank=2 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] avg=32768.0000 max=33096 max_rank=3 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=25 tokens_per_ep_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33096 max_rank=0 min=32585 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[32529, 32798, 32708, 32921, 32735, 32509, 32876, 33068] avg=32768.0000 max=33068 max_rank=7 min=32509 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=25 tokens_per_ep_rank=[33115, 32836, 32566, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33115 max_rank=0 min=32566 min_rank=2 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33108, 32694, 32384, 32849, 32568] avg=32768.0000 max=33108 max_rank=3 min=32384 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33108, 32694, 32384, 32849, 32568] avg=32768.0000 max=33108 max_rank=3 min=32384 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=25 tokens_per_ep_rank=[33115, 32836, 32566, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33115 max_rank=0 min=32566 min_rank=2 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=25 tokens_per_ep_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33096 max_rank=0 min=32585 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=25 tokens_per_ep_rank=[33115, 32836, 32566, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33115 max_rank=0 min=32566 min_rank=2 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=25 tokens_per_ep_rank=[33115, 32836, 32566, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33115 max_rank=0 min=32566 min_rank=2 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] avg=32768.0000 max=33096 max_rank=3 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[32837, 32700, 33004, 33096, 32694, 32396, 32849, 32568] avg=32768.0000 max=33096 max_rank=3 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=25 tokens_per_ep_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33096 max_rank=0 min=32585 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=25 tokens_per_ep_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33096 max_rank=0 min=32585 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=25 tokens_per_ep_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33096 max_rank=0 min=32585 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[32586, 32950, 32628, 32755, 32732, 32846, 32772, 32875] avg=32768.0000 max=32950 max_rank=1 min=32586 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=25 tokens_per_ep_rank=[33115, 32836, 32566, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33115 max_rank=0 min=32566 min_rank=2 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=25 tokens_per_ep_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33096 max_rank=0 min=32585 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[33018, 32947, 32802, 32989, 32574, 32556, 32805, 32453] avg=32768.0000 max=33018 max_rank=0 min=32453 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=29 tokens_per_ep_rank=[32521, 32622, 33111, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33111 max_rank=2 min=32521 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=25 tokens_per_ep_rank=[33115, 32836, 32566, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33115 max_rank=0 min=32566 min_rank=2 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=25 tokens_per_ep_rank=[33115, 32836, 32566, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33115 max_rank=0 min=32566 min_rank=2 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=29 tokens_per_ep_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33096 max_rank=2 min=32536 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=25 tokens_per_ep_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33096 max_rank=0 min=32585 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=25 tokens_per_ep_rank=[33096, 32836, 32585, 32811, 32696, 32637, 32634, 32849] avg=32768.0000 max=33096 max_rank=0 min=32585 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=29 tokens_per_ep_rank=[32521, 32622, 33111, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33111 max_rank=2 min=32521 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=29 tokens_per_ep_rank=[32521, 32622, 33111, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33111 max_rank=2 min=32521 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=29 tokens_per_ep_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33096 max_rank=2 min=32536 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=29 tokens_per_ep_rank=[32521, 32622, 33111, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33111 max_rank=2 min=32521 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=29 tokens_per_ep_rank=[32521, 32622, 33111, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33111 max_rank=2 min=32521 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=29 tokens_per_ep_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33096 max_rank=2 min=32536 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=26 tokens_per_ep_rank=[32780, 32992, 32717, 32753, 33018, 32584, 32766, 32534] avg=32768.0000 max=33018 max_rank=4 min=32534 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=29 tokens_per_ep_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33096 max_rank=2 min=32536 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=29 tokens_per_ep_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33096 max_rank=2 min=32536 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=27 tokens_per_ep_rank=[32557, 32788, 32762, 32765, 32467, 33006, 32938, 32861] avg=32768.0000 max=33006 max_rank=5 min=32467 min_rank=4 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33128, 32581, 32760, 32563, 32912] avg=32768.0000 max=33128 max_rank=3 min=32563 min_rank=6 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=29 tokens_per_ep_rank=[32521, 32622, 33111, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33111 max_rank=2 min=32521 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] avg=32768.0000 max=33096 max_rank=3 min=32581 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=29 tokens_per_ep_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33096 max_rank=2 min=32536 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33128, 32581, 32760, 32563, 32912] avg=32768.0000 max=33128 max_rank=3 min=32563 min_rank=6 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32244, 33052, 32698, 33196, 32739, 33027] avg=32768.0000 max=33196 max_rank=5 min=32244 min_rank=2 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=28 tokens_per_ep_rank=[32921, 32397, 32962, 32674, 32921, 32691, 32733, 32845] avg=32768.0000 max=32962 max_rank=2 min=32397 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] avg=32768.0000 max=33096 max_rank=3 min=32581 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33128, 32581, 32760, 32563, 32912] avg=32768.0000 max=33128 max_rank=3 min=32563 min_rank=6 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] avg=32768.0000 max=33096 max_rank=5 min=32344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=29 tokens_per_ep_rank=[32521, 32622, 33111, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33111 max_rank=2 min=32521 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33128, 32581, 32760, 32563, 32912] avg=32768.0000 max=33128 max_rank=3 min=32563 min_rank=6 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33128, 32581, 32760, 32563, 32912] avg=32768.0000 max=33128 max_rank=3 min=32563 min_rank=6 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] avg=32768.0000 max=33096 max_rank=3 min=32581 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=29 tokens_per_ep_rank=[32521, 32622, 33111, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33111 max_rank=2 min=32521 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=29 tokens_per_ep_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33096 max_rank=2 min=32536 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] avg=32768.0000 max=33096 max_rank=3 min=32581 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32244, 33052, 32698, 33196, 32739, 33027] avg=32768.0000 max=33196 max_rank=5 min=32244 min_rank=2 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] avg=32768.0000 max=33096 max_rank=3 min=32581 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=29 tokens_per_ep_rank=[32536, 32622, 33096, 32677, 32936, 32725, 32694, 32858] avg=32768.0000 max=33096 max_rank=2 min=32536 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] avg=32768.0000 max=33096 max_rank=5 min=32344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32244, 33052, 32698, 33196, 32739, 33027] avg=32768.0000 max=33196 max_rank=5 min=32244 min_rank=2 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32244, 33052, 32698, 33196, 32739, 33027] avg=32768.0000 max=33196 max_rank=5 min=32244 min_rank=2 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32244, 33052, 32698, 33196, 32739, 33027] avg=32768.0000 max=33196 max_rank=5 min=32244 min_rank=2 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] avg=32768.0000 max=33096 max_rank=5 min=32344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] avg=32768.0000 max=33096 max_rank=5 min=32344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] avg=32768.0000 max=33096 max_rank=5 min=32344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33128, 32581, 32760, 32563, 32912] avg=32768.0000 max=33128 max_rank=3 min=32563 min_rank=6 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=30 tokens_per_ep_rank=[32992, 32949, 32812, 32783, 32508, 32905, 32586, 32609] avg=32768.0000 max=32992 max_rank=0 min=32508 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] avg=32768.0000 max=33096 max_rank=3 min=32581 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32244, 33052, 32698, 33196, 32739, 33027] avg=32768.0000 max=33196 max_rank=5 min=32244 min_rank=2 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=31 tokens_per_ep_rank=[32609, 32915, 32871, 32470, 32554, 33020, 32960, 32745] avg=32768.0000 max=33020 max_rank=5 min=32470 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33128, 32581, 32760, 32563, 32912] avg=32768.0000 max=33128 max_rank=3 min=32563 min_rank=6 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] avg=32768.0000 max=33096 max_rank=5 min=32344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33128, 32581, 32760, 32563, 32912] avg=32768.0000 max=33128 max_rank=3 min=32563 min_rank=6 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] avg=32768.0000 max=33096 max_rank=3 min=32581 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=32 tokens_per_ep_rank=[32669, 32763, 32768, 33096, 32581, 32760, 32595, 32912] avg=32768.0000 max=33096 max_rank=3 min=32581 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32244, 33052, 32698, 33196, 32739, 33027] avg=32768.0000 max=33196 max_rank=5 min=32244 min_rank=2 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] avg=32768.0000 max=33096 max_rank=5 min=32344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32244, 33052, 32698, 33196, 32739, 33027] avg=32768.0000 max=33196 max_rank=5 min=32244 min_rank=2 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=33 tokens_per_ep_rank=[32738, 32450, 32344, 33052, 32698, 33096, 32739, 33027] avg=32768.0000 max=33096 max_rank=5 min=32344 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33103, 32843, 32715, 32515, 32773] avg=32768.0000 max=33103 max_rank=3 min=32515 min_rank=6 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] avg=32768.0000 max=33096 max_rank=3 min=32522 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=34 tokens_per_ep_rank=[32718, 32694, 32906, 32889, 32689, 32858, 32575, 32815] avg=32768.0000 max=32906 max_rank=2 min=32575 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33103, 32843, 32715, 32515, 32773] avg=32768.0000 max=33103 max_rank=3 min=32515 min_rank=6 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] avg=32768.0000 max=33096 max_rank=3 min=32522 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33103, 32843, 32715, 32515, 32773] avg=32768.0000 max=33103 max_rank=3 min=32515 min_rank=6 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33103, 32843, 32715, 32515, 32773] avg=32768.0000 max=33103 max_rank=3 min=32515 min_rank=6 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=35 tokens_per_ep_rank=[32460, 33049, 32815, 32762, 32748, 32714, 32617, 32979] avg=32768.0000 max=33049 max_rank=1 min=32460 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33103, 32843, 32715, 32515, 32773] avg=32768.0000 max=33103 max_rank=3 min=32515 min_rank=6 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] avg=32768.0000 max=33096 max_rank=3 min=32522 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] avg=32768.0000 max=33096 max_rank=3 min=32522 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33196, 32726, 32501, 32711, 32552, 32803] avg=32768.0000 max=33196 max_rank=2 min=32501 min_rank=4 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] avg=32768.0000 max=33096 max_rank=3 min=32522 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] avg=32768.0000 max=33096 max_rank=2 min=32552 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=36 tokens_per_ep_rank=[32986, 32581, 32844, 32885, 32721, 32886, 32727, 32514] avg=32768.0000 max=32986 max_rank=0 min=32514 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33196, 32726, 32501, 32711, 32552, 32803] avg=32768.0000 max=33196 max_rank=2 min=32501 min_rank=4 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33196, 32726, 32501, 32711, 32552, 32803] avg=32768.0000 max=33196 max_rank=2 min=32501 min_rank=4 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33103, 32843, 32715, 32515, 32773] avg=32768.0000 max=33103 max_rank=3 min=32515 min_rank=6 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] avg=32768.0000 max=33096 max_rank=2 min=32552 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33196, 32726, 32501, 32711, 32552, 32803] avg=32768.0000 max=33196 max_rank=2 min=32501 min_rank=4 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33196, 32726, 32501, 32711, 32552, 32803] avg=32768.0000 max=33196 max_rank=2 min=32501 min_rank=4 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] avg=32768.0000 max=33096 max_rank=2 min=32552 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=37 tokens_per_ep_rank=[32746, 32981, 32730, 33045, 32555, 32688, 32654, 32745] avg=32768.0000 max=33045 max_rank=3 min=32555 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] avg=32768.0000 max=33096 max_rank=2 min=32552 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] avg=32768.0000 max=33096 max_rank=3 min=32522 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] avg=32768.0000 max=33096 max_rank=2 min=32552 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33103, 32843, 32715, 32515, 32773] avg=32768.0000 max=33103 max_rank=3 min=32515 min_rank=6 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=38 tokens_per_ep_rank=[32961, 32620, 32670, 32604, 32962, 32824, 32815, 32688] avg=32768.0000 max=32962 max_rank=4 min=32604 min_rank=3 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] avg=32768.0000 max=33096 max_rank=3 min=32522 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33103, 32843, 32715, 32515, 32773] avg=32768.0000 max=33103 max_rank=3 min=32515 min_rank=6 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33196, 32726, 32501, 32711, 32552, 32803] avg=32768.0000 max=33196 max_rank=2 min=32501 min_rank=4 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=39 tokens_per_ep_rank=[32819, 32634, 32742, 33096, 32843, 32715, 32522, 32773] avg=32768.0000 max=33096 max_rank=3 min=32522 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] avg=32768.0000 max=33096 max_rank=2 min=32552 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33196, 32726, 32501, 32711, 32552, 32803] avg=32768.0000 max=33196 max_rank=2 min=32501 min_rank=4 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=40 tokens_per_ep_rank=[32639, 32738, 32811, 32745, 32722, 32778, 32832, 32879] avg=32768.0000 max=32879 max_rank=7 min=32639 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] avg=32768.0000 max=33096 max_rank=2 min=32552 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33196, 32726, 32501, 32711, 32552, 32803] avg=32768.0000 max=33196 max_rank=2 min=32501 min_rank=4 imbalance_ratio=1.013062 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33112, 32645, 32886, 32902, 32583, 32554] avg=32768.0000 max=33112 max_rank=2 min=32554 min_rank=7 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=41 tokens_per_ep_rank=[32789, 32866, 33096, 32726, 32601, 32711, 32552, 32803] avg=32768.0000 max=33096 max_rank=2 min=32552 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=100 max_load_reduction_pct=0.003012 moved_to_replicas=100 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] avg=32768.0000 max=33096 max_rank=2 min=32570 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33140, 32561, 32812, 32595] avg=32768.0000 max=33140 max_rank=4 min=32561 min_rank=5 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33112, 32645, 32886, 32902, 32583, 32554] avg=32768.0000 max=33112 max_rank=2 min=32554 min_rank=7 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=42 tokens_per_ep_rank=[32828, 32845, 32691, 32738, 32803, 32681, 32845, 32713] avg=32768.0000 max=32845 max_rank=1 min=32681 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33112, 32645, 32886, 32902, 32583, 32554] avg=32768.0000 max=33112 max_rank=2 min=32554 min_rank=7 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] avg=32768.0000 max=33096 max_rank=4 min=32595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] avg=32768.0000 max=33096 max_rank=2 min=32570 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33112, 32645, 32886, 32902, 32583, 32554] avg=32768.0000 max=33112 max_rank=2 min=32554 min_rank=7 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] avg=32768.0000 max=33096 max_rank=2 min=32570 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33112, 32645, 32886, 32902, 32583, 32554] avg=32768.0000 max=33112 max_rank=2 min=32554 min_rank=7 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] avg=32768.0000 max=33096 max_rank=2 min=32570 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33140, 32561, 32812, 32595] avg=32768.0000 max=33140 max_rank=4 min=32561 min_rank=5 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=43 tokens_per_ep_rank=[33000, 32663, 32970, 32989, 32704, 32806, 32296, 32716] avg=32768.0000 max=33000 max_rank=0 min=32296 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] avg=32768.0000 max=33096 max_rank=2 min=32570 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33140, 32561, 32812, 32595] avg=32768.0000 max=33140 max_rank=4 min=32561 min_rank=5 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] avg=32768.0000 max=33096 max_rank=4 min=32595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33140, 32561, 32812, 32595] avg=32768.0000 max=33140 max_rank=4 min=32561 min_rank=5 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] avg=32768.0000 max=33096 max_rank=4 min=32595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33140, 32561, 32812, 32595] avg=32768.0000 max=33140 max_rank=4 min=32561 min_rank=5 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] avg=32768.0000 max=33096 max_rank=4 min=32595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=44 tokens_per_ep_rank=[32916, 32560, 32769, 32953, 32683, 32846, 32753, 32664] avg=32768.0000 max=32953 max_rank=3 min=32560 min_rank=1 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] avg=32768.0000 max=33096 max_rank=4 min=32595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33112, 32645, 32886, 32902, 32583, 32554] avg=32768.0000 max=33112 max_rank=2 min=32554 min_rank=7 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] avg=32768.0000 max=33096 max_rank=2 min=32570 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=45 tokens_per_ep_rank=[32647, 32770, 32811, 32916, 33061, 32639, 32753, 32547] avg=32768.0000 max=33061 max_rank=4 min=32547 min_rank=7 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33140, 32561, 32812, 32595] avg=32768.0000 max=33140 max_rank=4 min=32561 min_rank=5 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] avg=32768.0000 max=33096 max_rank=4 min=32595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33112, 32645, 32886, 32902, 32583, 32554] avg=32768.0000 max=33112 max_rank=2 min=32554 min_rank=7 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=46 tokens_per_ep_rank=[32749, 32864, 32754, 32767, 32672, 32626, 32922, 32790] avg=32768.0000 max=32922 max_rank=6 min=32626 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] avg=32768.0000 max=33096 max_rank=2 min=32570 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33112, 32645, 32886, 32902, 32583, 32554] avg=32768.0000 max=33112 max_rank=2 min=32554 min_rank=7 imbalance_ratio=1.010498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33140, 32561, 32812, 32595] avg=32768.0000 max=33140 max_rank=4 min=32561 min_rank=5 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=47 tokens_per_ep_rank=[32853, 32609, 33096, 32645, 32886, 32902, 32583, 32570] avg=32768.0000 max=33096 max_rank=2 min=32570 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000483 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] avg=32768.0000 max=33096 max_rank=4 min=32595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33140, 32561, 32812, 32595] avg=32768.0000 max=33140 max_rank=4 min=32561 min_rank=5 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=48 tokens_per_ep_rank=[32650, 32688, 32834, 32864, 33096, 32605, 32812, 32595] avg=32768.0000 max=33096 max_rank=4 min=32595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=48 total_expected_recv=1570939 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346752 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=48 total_expected_recv=1572547 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344448 peak_demand=33068 required_capacity=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=48 total_expected_recv=1571733 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346112 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=48 total_expected_recv=1571828 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346752 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=48 total_expected_recv=1574797 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343168 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=48 total_expected_recv=1572846 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344576 peak_demand=33049 required_capacity=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=48 total_expected_recv=1574058 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342272 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=48 total_expected_recv=1574164 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341760 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:48:15] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 1259.9 | throughput per GPU (TFLOP/s/GPU): 70.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095574E+01 | loss scale: 1.0 | grad norm: 56.703 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32468 expected_overflow=0 exact_recv_per_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32468 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33050 expected_overflow=0 exact_recv_per_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33050 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33087 expected_overflow=0 exact_recv_per_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33020 expected_overflow=0 exact_recv_per_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32373 expected_overflow=0 exact_recv_per_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33087 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=33020 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32373 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32439 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32915 expected_overflow=0 exact_recv_per_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32915 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33085 expected_overflow=0 exact_recv_per_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=40960 expected_recv=33085 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33041 expected_overflow=0 exact_recv_per_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=40960 expected_recv=33041 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32959 expected_overflow=0 exact_recv_per_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32399 expected_overflow=0 exact_recv_per_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32959 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32399 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33008 expected_overflow=0 exact_recv_per_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=40960 expected_recv=33008 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32957 expected_overflow=0 exact_recv_per_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32957 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32416 expected_overflow=0 exact_recv_per_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32416 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32973 expected_overflow=0 exact_recv_per_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32973 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32945 expected_overflow=0 exact_recv_per_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32945 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33043 expected_overflow=0 exact_recv_per_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32995 expected_overflow=0 exact_recv_per_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=40960 expected_recv=33043 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32995 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=23 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33009 expected_overflow=0 exact_recv_per_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=25 calls=1 budget=40960 expected_recv=33009 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32601 expected_overflow=0 exact_recv_per_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=24 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=24 calls=1 budget=40960 expected_recv=32601 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=25 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=26 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33088 expected_overflow=0 exact_recv_per_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33088 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=28 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33083 expected_overflow=0 exact_recv_per_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=27 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=27 calls=1 budget=40960 expected_recv=33083 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32517 expected_overflow=0 exact_recv_per_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32517 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=28 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32376 expected_overflow=0 exact_recv_per_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32915 expected_overflow=0 exact_recv_per_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33083 expected_overflow=0 exact_recv_per_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32376 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32915 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33083 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=29 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32975 expected_overflow=0 exact_recv_per_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32975 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32914 expected_overflow=0 exact_recv_per_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33019 expected_overflow=0 exact_recv_per_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=30 calls=1 budget=40960 expected_recv=32914 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32381 expected_overflow=0 exact_recv_per_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=31 calls=1 budget=40960 expected_recv=33019 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32381 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32948 expected_overflow=0 exact_recv_per_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32948 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=31 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32595 expected_overflow=0 exact_recv_per_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32595 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=32 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=33 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33076 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33076 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32282 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=34 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=35 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=36 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32990 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=37 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32510 expected_overflow=0 exact_recv_per_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32510 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=38 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32629 expected_overflow=0 exact_recv_per_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32960 expected_overflow=0 exact_recv_per_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32629 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32960 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32948 expected_overflow=0 exact_recv_per_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32991 expected_overflow=0 exact_recv_per_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32948 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32991 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=39 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33000 expected_overflow=0 exact_recv_per_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=42 calls=1 budget=40960 expected_recv=33000 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=40 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32452 expected_overflow=0 exact_recv_per_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32452 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=41 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32385 expected_overflow=0 exact_recv_per_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32385 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32510 expected_overflow=0 exact_recv_per_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32510 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=42 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33009 expected_overflow=0 exact_recv_per_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33009 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33073 expected_overflow=0 exact_recv_per_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=45 calls=1 budget=40960 expected_recv=33073 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=43 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32948 expected_overflow=0 exact_recv_per_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32948 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=44 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=45 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=46 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=47 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32657, 32564, 32819, 32908, 32685, 32839, 32730, 32942] avg=32768.0000 max=32942 max_rank=7 min=32564 min_rank=1 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32860, 32751, 32887, 32961, 32807, 32986, 32496, 32396] avg=32768.0000 max=32986 max_rank=5 min=32396 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32761, 32731, 32853, 32700, 32806, 32738, 32894, 32661] avg=32768.0000 max=32894 max_rank=6 min=32661 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[32658, 32968, 32763, 32468, 32965, 32620, 32763, 32939] avg=32768.0000 max=32968 max_rank=1 min=32468 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32621, 32781, 32796, 32764, 32649, 32701, 32853, 32979] avg=32768.0000 max=32979 max_rank=7 min=32621 min_rank=0 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32645, 32749, 32573, 32793, 32667, 32810, 33053, 32854] avg=32768.0000 max=33053 max_rank=6 min=32573 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32772, 32759, 32589, 33050, 32809, 32564, 32872, 32729] avg=32768.0000 max=33050 max_rank=3 min=32564 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[33087, 32373, 32635, 32661, 32979, 32946, 33020, 32443] avg=32768.0000 max=33087 max_rank=0 min=32373 min_rank=1 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[32848, 32664, 32610, 33005, 32760, 33010, 32419, 32828] avg=32768.0000 max=33010 max_rank=5 min=32419 min_rank=6 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[32692, 32942, 32439, 32709, 32504, 33065, 32878, 32915] avg=32768.0000 max=33065 max_rank=5 min=32439 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[32746, 32574, 32611, 33085, 32860, 32375, 32966, 32927] avg=32768.0000 max=33085 max_rank=3 min=32375 min_rank=5 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[32838, 32732, 32702, 32711, 32881, 32821, 32513, 32946] avg=32768.0000 max=32946 max_rank=7 min=32513 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[32539, 32653, 32672, 32775, 33041, 32865, 32959, 32640] avg=32768.0000 max=33041 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[32921, 33008, 32399, 32780, 32491, 32866, 32717, 32962] avg=32768.0000 max=33008 max_rank=1 min=32399 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[32786, 32717, 32899, 32537, 32713, 33072, 32669, 32751] avg=32768.0000 max=33072 max_rank=5 min=32537 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[32793, 32801, 32765, 32766, 32475, 32927, 32660, 32957] avg=32768.0000 max=32957 max_rank=7 min=32475 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[32951, 32892, 32704, 32675, 32892, 32856, 32758, 32416] avg=32768.0000 max=32951 max_rank=0 min=32416 min_rank=7 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[32782, 32707, 32693, 32735, 32801, 32727, 32885, 32814] avg=32768.0000 max=32885 max_rank=6 min=32693 min_rank=2 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[32609, 32628, 32953, 32590, 32618, 32806, 32967, 32973] avg=32768.0000 max=32973 max_rank=7 min=32590 min_rank=3 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33116, 32659, 32601, 32595, 32911, 32793, 32745] avg=32768.0000 max=33116 max_rank=1 min=32595 min_rank=4 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[32707, 32945, 32768, 32665, 32860, 32739, 32614, 32846] avg=32768.0000 max=32945 max_rank=1 min=32614 min_rank=6 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33116, 32659, 32601, 32595, 32911, 32793, 32745] avg=32768.0000 max=33116 max_rank=1 min=32595 min_rank=4 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] avg=32768.0000 max=33096 max_rank=1 min=32601 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33116, 32659, 32601, 32595, 32911, 32793, 32745] avg=32768.0000 max=33116 max_rank=1 min=32595 min_rank=4 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] avg=32768.0000 max=33096 max_rank=1 min=32601 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33116, 32659, 32601, 32595, 32911, 32793, 32745] avg=32768.0000 max=33116 max_rank=1 min=32595 min_rank=4 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] avg=32768.0000 max=33096 max_rank=1 min=32601 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33116, 32659, 32601, 32595, 32911, 32793, 32745] avg=32768.0000 max=33116 max_rank=1 min=32595 min_rank=4 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] avg=32768.0000 max=33096 max_rank=1 min=32601 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[32843, 32850, 32537, 32563, 32539, 33043, 32995, 32774] avg=32768.0000 max=33043 max_rank=5 min=32537 min_rank=2 imbalance_ratio=1.008392 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] avg=32768.0000 max=33096 max_rank=1 min=32601 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=27 tokens_per_ep_rank=[33200, 32758, 32738, 32693, 32707, 32583, 32813, 32652] avg=32768.0000 max=33200 max_rank=0 min=32583 min_rank=5 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=27 tokens_per_ep_rank=[33200, 32758, 32738, 32693, 32707, 32583, 32813, 32652] avg=32768.0000 max=33200 max_rank=0 min=32583 min_rank=5 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[32609, 33058, 32659, 32762, 32836, 32654, 32612, 32954] avg=32768.0000 max=33058 max_rank=1 min=32609 min_rank=0 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=27 tokens_per_ep_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] avg=32768.0000 max=33096 max_rank=0 min=32652 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=27 tokens_per_ep_rank=[33200, 32758, 32738, 32693, 32707, 32583, 32813, 32652] avg=32768.0000 max=33200 max_rank=0 min=32583 min_rank=5 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=27 tokens_per_ep_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] avg=32768.0000 max=33096 max_rank=0 min=32652 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=27 tokens_per_ep_rank=[33200, 32758, 32738, 32693, 32707, 32583, 32813, 32652] avg=32768.0000 max=33200 max_rank=0 min=32583 min_rank=5 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=27 tokens_per_ep_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] avg=32768.0000 max=33096 max_rank=0 min=32652 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=27 tokens_per_ep_rank=[33200, 32758, 32738, 32693, 32707, 32583, 32813, 32652] avg=32768.0000 max=33200 max_rank=0 min=32583 min_rank=5 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33116, 32659, 32601, 32595, 32911, 32793, 32745] avg=32768.0000 max=33116 max_rank=1 min=32595 min_rank=4 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=27 tokens_per_ep_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] avg=32768.0000 max=33096 max_rank=0 min=32652 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=27 tokens_per_ep_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] avg=32768.0000 max=33096 max_rank=0 min=32652 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[32813, 32744, 32831, 32791, 32664, 32785, 32904, 32612] avg=32768.0000 max=32904 max_rank=6 min=32612 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] avg=32768.0000 max=33096 max_rank=1 min=32601 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33116, 32659, 32601, 32595, 32911, 32793, 32745] avg=32768.0000 max=33116 max_rank=1 min=32595 min_rank=4 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[32952, 32922, 32531, 32814, 32721, 32562, 32799, 32843] avg=32768.0000 max=32952 max_rank=0 min=32531 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] avg=32768.0000 max=33096 max_rank=1 min=32601 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=27 tokens_per_ep_rank=[33200, 32758, 32738, 32693, 32707, 32583, 32813, 32652] avg=32768.0000 max=33200 max_rank=0 min=32583 min_rank=5 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33116, 32659, 32601, 32595, 32911, 32793, 32745] avg=32768.0000 max=33116 max_rank=1 min=32595 min_rank=4 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=27 tokens_per_ep_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] avg=32768.0000 max=33096 max_rank=0 min=32652 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=25 tokens_per_ep_rank=[32724, 33096, 32659, 32601, 32615, 32911, 32793, 32745] avg=32768.0000 max=33096 max_rank=1 min=32601 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=27 tokens_per_ep_rank=[33200, 32758, 32738, 32693, 32707, 32583, 32813, 32652] avg=32768.0000 max=33200 max_rank=0 min=32583 min_rank=5 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=26 tokens_per_ep_rank=[32839, 32942, 32628, 32641, 32719, 32808, 33009, 32558] avg=32768.0000 max=33009 max_rank=6 min=32558 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=27 tokens_per_ep_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] avg=32768.0000 max=33096 max_rank=0 min=32652 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=27 tokens_per_ep_rank=[33200, 32758, 32738, 32693, 32707, 32583, 32813, 32652] avg=32768.0000 max=33200 max_rank=0 min=32583 min_rank=5 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=27 tokens_per_ep_rank=[33096, 32758, 32738, 32693, 32707, 32687, 32813, 32652] avg=32768.0000 max=33096 max_rank=0 min=32652 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=28 tokens_per_ep_rank=[32724, 32656, 32992, 33083, 32669, 32518, 32712, 32790] avg=32768.0000 max=33083 max_rank=3 min=32518 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=29 tokens_per_ep_rank=[32517, 32466, 32732, 32769, 32823, 33088, 33028, 32721] avg=32768.0000 max=33088 max_rank=5 min=32466 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=30 tokens_per_ep_rank=[32872, 32889, 32965, 32529, 32758, 32940, 32661, 32530] avg=32768.0000 max=32965 max_rank=2 min=32529 min_rank=3 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=36 tokens_per_ep_rank=[32576, 32617, 32817, 32886, 32769, 32577, 33102, 32800] avg=32768.0000 max=33102 max_rank=6 min=32576 min_rank=0 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=36 tokens_per_ep_rank=[32576, 32617, 32817, 32886, 32769, 32577, 33102, 32800] avg=32768.0000 max=33102 max_rank=6 min=32576 min_rank=0 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=36 tokens_per_ep_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] avg=32768.0000 max=33096 max_rank=6 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=36 tokens_per_ep_rank=[32576, 32617, 32817, 32886, 32769, 32577, 33102, 32800] avg=32768.0000 max=33102 max_rank=6 min=32576 min_rank=0 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=36 tokens_per_ep_rank=[32576, 32617, 32817, 32886, 32769, 32577, 33102, 32800] avg=32768.0000 max=33102 max_rank=6 min=32576 min_rank=0 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=36 tokens_per_ep_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] avg=32768.0000 max=33096 max_rank=6 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=36 tokens_per_ep_rank=[32576, 32617, 32817, 32886, 32769, 32577, 33102, 32800] avg=32768.0000 max=33102 max_rank=6 min=32576 min_rank=0 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=31 tokens_per_ep_rank=[32790, 32975, 32721, 32914, 32818, 32915, 32376, 32635] avg=32768.0000 max=32975 max_rank=1 min=32376 min_rank=6 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=36 tokens_per_ep_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] avg=32768.0000 max=33096 max_rank=6 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=36 tokens_per_ep_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] avg=32768.0000 max=33096 max_rank=6 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=37 tokens_per_ep_rank=[33150, 32852, 32826, 32522, 33076, 32704, 32228, 32786] avg=32768.0000 max=33150 max_rank=0 min=32228 min_rank=6 imbalance_ratio=1.011658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=36 tokens_per_ep_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] avg=32768.0000 max=33096 max_rank=6 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=37 tokens_per_ep_rank=[33150, 32852, 32826, 32522, 33076, 32704, 32228, 32786] avg=32768.0000 max=33150 max_rank=0 min=32228 min_rank=6 imbalance_ratio=1.011658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=37 tokens_per_ep_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54 max_load_reduction_pct=0.001629 moved_to_replicas=54 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=37 tokens_per_ep_rank=[33150, 32852, 32826, 32522, 33076, 32704, 32228, 32786] avg=32768.0000 max=33150 max_rank=0 min=32228 min_rank=6 imbalance_ratio=1.011658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=37 tokens_per_ep_rank=[33150, 32852, 32826, 32522, 33076, 32704, 32228, 32786] avg=32768.0000 max=33150 max_rank=0 min=32228 min_rank=6 imbalance_ratio=1.011658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=37 tokens_per_ep_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54 max_load_reduction_pct=0.001629 moved_to_replicas=54 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=37 tokens_per_ep_rank=[33150, 32852, 32826, 32522, 33076, 32704, 32228, 32786] avg=32768.0000 max=33150 max_rank=0 min=32228 min_rank=6 imbalance_ratio=1.011658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=32 tokens_per_ep_rank=[33019, 32775, 33083, 32735, 32528, 32575, 32523, 32906] avg=32768.0000 max=33083 max_rank=2 min=32523 min_rank=6 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=37 tokens_per_ep_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54 max_load_reduction_pct=0.001629 moved_to_replicas=54 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=37 tokens_per_ep_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54 max_load_reduction_pct=0.001629 moved_to_replicas=54 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=37 tokens_per_ep_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54 max_load_reduction_pct=0.001629 moved_to_replicas=54 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=33 tokens_per_ep_rank=[32734, 32813, 32739, 32891, 32966, 32522, 32890, 32589] avg=32768.0000 max=32966 max_rank=4 min=32522 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=36 tokens_per_ep_rank=[32576, 32617, 32817, 32886, 32769, 32577, 33102, 32800] avg=32768.0000 max=33102 max_rank=6 min=32576 min_rank=0 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=36 tokens_per_ep_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] avg=32768.0000 max=33096 max_rank=6 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=34 tokens_per_ep_rank=[32595, 32919, 32381, 32955, 32741, 32846, 32759, 32948] avg=32768.0000 max=32955 max_rank=3 min=32381 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=37 tokens_per_ep_rank=[33150, 32852, 32826, 32522, 33076, 32704, 32228, 32786] avg=32768.0000 max=33150 max_rank=0 min=32228 min_rank=6 imbalance_ratio=1.011658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=36 tokens_per_ep_rank=[32576, 32617, 32817, 32886, 32769, 32577, 33102, 32800] avg=32768.0000 max=33102 max_rank=6 min=32576 min_rank=0 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=37 tokens_per_ep_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54 max_load_reduction_pct=0.001629 moved_to_replicas=54 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=35 tokens_per_ep_rank=[32722, 32688, 32971, 32763, 32599, 32732, 32940, 32729] avg=32768.0000 max=32971 max_rank=2 min=32599 min_rank=4 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=36 tokens_per_ep_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] avg=32768.0000 max=33096 max_rank=6 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=36 tokens_per_ep_rank=[32576, 32617, 32817, 32886, 32769, 32577, 33102, 32800] avg=32768.0000 max=33102 max_rank=6 min=32576 min_rank=0 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=37 tokens_per_ep_rank=[33150, 32852, 32826, 32522, 33076, 32704, 32228, 32786] avg=32768.0000 max=33150 max_rank=0 min=32228 min_rank=6 imbalance_ratio=1.011658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=36 tokens_per_ep_rank=[32582, 32617, 32817, 32886, 32769, 32577, 33096, 32800] avg=32768.0000 max=33096 max_rank=6 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=37 tokens_per_ep_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54 max_load_reduction_pct=0.001629 moved_to_replicas=54 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=37 tokens_per_ep_rank=[33150, 32852, 32826, 32522, 33076, 32704, 32228, 32786] avg=32768.0000 max=33150 max_rank=0 min=32228 min_rank=6 imbalance_ratio=1.011658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=37 tokens_per_ep_rank=[33096, 32852, 32826, 32522, 33076, 32704, 32282, 32786] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=54 max_load_reduction_pct=0.001629 moved_to_replicas=54 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=38 tokens_per_ep_rank=[32865, 32990, 32792, 32784, 32694, 32691, 32620, 32708] avg=32768.0000 max=32990 max_rank=1 min=32620 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=39 tokens_per_ep_rank=[32762, 32659, 32869, 32865, 32796, 32756, 32696, 32741] avg=32768.0000 max=32869 max_rank=2 min=32659 min_rank=1 imbalance_ratio=1.003082 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=40 tokens_per_ep_rank=[32932, 32637, 32681, 32754, 32801, 32738, 32790, 32811] avg=32768.0000 max=32932 max_rank=0 min=32637 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=41 tokens_per_ep_rank=[32991, 32769, 32751, 32793, 32984, 32510, 32771, 32575] avg=32768.0000 max=32991 max_rank=0 min=32510 min_rank=5 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=42 tokens_per_ep_rank=[32891, 32713, 32911, 32624, 32629, 32778, 32960, 32638] avg=32768.0000 max=32960 max_rank=6 min=32624 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=43 tokens_per_ep_rank=[32487, 32452, 33024, 32781, 32681, 32771, 33000, 32948] avg=32768.0000 max=33024 max_rank=2 min=32452 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=44 tokens_per_ep_rank=[32803, 32704, 32636, 32737, 32894, 32717, 32854, 32799] avg=32768.0000 max=32894 max_rank=4 min=32636 min_rank=2 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=45 tokens_per_ep_rank=[32892, 32928, 32687, 32630, 32902, 32768, 32660, 32677] avg=32768.0000 max=32928 max_rank=1 min=32630 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=46 tokens_per_ep_rank=[32922, 32913, 32385, 32675, 33009, 33073, 32657, 32510] avg=32768.0000 max=33073 max_rank=5 min=32385 min_rank=2 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=47 tokens_per_ep_rank=[32831, 32858, 32831, 32908, 32751, 32756, 32705, 32504] avg=32768.0000 max=32908 max_rank=3 min=32504 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=48 tokens_per_ep_rank=[32589, 32856, 32948, 32820, 32612, 32720, 32670, 32929] avg=32768.0000 max=32948 max_rank=2 min=32589 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=48 total_expected_recv=1571455 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346112 peak_demand=33083 required_capacity=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=48 total_expected_recv=1573261 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=48 total_expected_recv=1572330 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344576 peak_demand=32979 required_capacity=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=48 total_expected_recv=1572564 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344320 peak_demand=33076 required_capacity=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=48 total_expected_recv=1573478 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344320 peak_demand=33088 required_capacity=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=48 total_expected_recv=1573438 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344192 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=48 total_expected_recv=1573765 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345088 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=48 total_expected_recv=1572621 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345088 peak_demand=33085 required_capacity=1.009674 + [2026-08-26 15:48:16] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 995.4 | throughput per GPU (TFLOP/s/GPU): 89.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.006465E+01 | loss scale: 1.0 | grad norm: 170.036 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32601 expected_overflow=0 exact_recv_per_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32601 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32536 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32926 expected_overflow=0 exact_recv_per_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32926 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32619 expected_overflow=0 exact_recv_per_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32936 expected_overflow=0 exact_recv_per_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32619 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32936 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32926 expected_overflow=0 exact_recv_per_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32990 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32926 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32915 expected_overflow=0 exact_recv_per_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32915 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33041 expected_overflow=0 exact_recv_per_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33041 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33091 expected_overflow=0 exact_recv_per_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=33091 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32950 expected_overflow=0 exact_recv_per_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32462 expected_overflow=0 exact_recv_per_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32915 expected_overflow=0 exact_recv_per_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32950 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32462 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=40960 expected_recv=32915 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32507 expected_overflow=0 exact_recv_per_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32507 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32495 expected_overflow=0 exact_recv_per_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32495 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33090 expected_overflow=0 exact_recv_per_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=40960 expected_recv=33090 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32419 expected_overflow=0 exact_recv_per_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32419 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32557 expected_overflow=0 exact_recv_per_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32557 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33052 expected_overflow=0 exact_recv_per_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=40960 expected_recv=33052 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32428 expected_overflow=0 exact_recv_per_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32428 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32991 expected_overflow=0 exact_recv_per_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32991 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32936 expected_overflow=0 exact_recv_per_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=40960 expected_recv=32936 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33049 expected_overflow=0 exact_recv_per_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=40960 expected_recv=33049 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32498 expected_overflow=0 exact_recv_per_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32498 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32541 expected_overflow=0 exact_recv_per_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=23 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32541 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32950 expected_overflow=0 exact_recv_per_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32950 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=23 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=24 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=25 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32991 expected_overflow=0 exact_recv_per_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32991 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32524 expected_overflow=0 exact_recv_per_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32524 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=26 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33029 expected_overflow=0 exact_recv_per_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=27 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=28 calls=1 budget=40960 expected_recv=33029 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=29 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=28 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32224 expected_overflow=0 exact_recv_per_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32224 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32997 expected_overflow=0 exact_recv_per_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=29 calls=1 budget=40960 expected_recv=32997 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=30 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32947 expected_overflow=0 exact_recv_per_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=31 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32947 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32969 expected_overflow=0 exact_recv_per_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32969 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=32 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=33 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32541 expected_overflow=0 exact_recv_per_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32541 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=34 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32459 expected_overflow=0 exact_recv_per_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32459 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=35 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=36 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=37 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=38 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32511 expected_overflow=0 exact_recv_per_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32511 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=39 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32957 expected_overflow=0 exact_recv_per_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32957 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=40 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=41 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=42 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=43 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=44 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=44 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=45 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=47 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=46 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=47 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[32844, 32601, 32785, 32712, 32794, 32911, 32961, 32536] avg=32768.0000 max=32961 max_rank=6 min=32536 min_rank=7 imbalance_ratio=1.005890 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32878, 32848, 32718, 32714, 32794, 32682, 32701, 32809] avg=32768.0000 max=32878 max_rank=0 min=32682 min_rank=5 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[32378, 32990, 32586, 33166, 32801, 32466, 32926, 32831] avg=32768.0000 max=33166 max_rank=3 min=32378 min_rank=0 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[32378, 32990, 32586, 33166, 32801, 32466, 32926, 32831] avg=32768.0000 max=33166 max_rank=3 min=32378 min_rank=0 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[32378, 32990, 32586, 33166, 32801, 32466, 32926, 32831] avg=32768.0000 max=33166 max_rank=3 min=32378 min_rank=0 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] avg=32768.0000 max=33096 max_rank=3 min=32448 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[32378, 32990, 32586, 33166, 32801, 32466, 32926, 32831] avg=32768.0000 max=33166 max_rank=3 min=32378 min_rank=0 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[32378, 32990, 32586, 33166, 32801, 32466, 32926, 32831] avg=32768.0000 max=33166 max_rank=3 min=32378 min_rank=0 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32928, 32654, 32641, 32926, 32774, 32854, 32542, 32825] avg=32768.0000 max=32928 max_rank=0 min=32542 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] avg=32768.0000 max=33096 max_rank=3 min=32448 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] avg=32768.0000 max=33096 max_rank=3 min=32448 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[32378, 32990, 32586, 33166, 32801, 32466, 32926, 32831] avg=32768.0000 max=33166 max_rank=3 min=32378 min_rank=0 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] avg=32768.0000 max=33096 max_rank=3 min=32448 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] avg=32768.0000 max=33096 max_rank=3 min=32448 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] avg=32768.0000 max=33096 max_rank=3 min=32448 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[32378, 32990, 32586, 33166, 32801, 32466, 32926, 32831] avg=32768.0000 max=33166 max_rank=3 min=32378 min_rank=0 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32651, 32764, 32699, 32850, 32619, 32780, 32936, 32845] avg=32768.0000 max=32936 max_rank=6 min=32619 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] avg=32768.0000 max=33096 max_rank=3 min=32448 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[32378, 32990, 32586, 33166, 32801, 32466, 32926, 32831] avg=32768.0000 max=33166 max_rank=3 min=32378 min_rank=0 imbalance_ratio=1.012146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[32448, 32990, 32586, 33096, 32801, 32466, 32926, 32831] avg=32768.0000 max=33096 max_rank=3 min=32448 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=70 max_load_reduction_pct=0.002111 moved_to_replicas=70 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32915, 32900, 32920, 32831, 32560, 32572, 32678, 32768] avg=32768.0000 max=32920 max_rank=2 min=32560 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32815, 32994, 32431, 32801, 32705, 33035, 32821, 32542] avg=32768.0000 max=33035 max_rank=5 min=32431 min_rank=2 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32999, 32778, 32590, 32818, 32587, 32927, 32703, 32742] avg=32768.0000 max=32999 max_rank=0 min=32587 min_rank=4 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32715, 32738, 32810, 32530, 32999, 32657, 32953, 32742] avg=32768.0000 max=32999 max_rank=4 min=32530 min_rank=3 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32533, 32401, 33041, 33091, 32894, 32700, 32756, 32728] avg=32768.0000 max=33091 max_rank=3 min=32401 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32802, 33005, 32689, 32815, 32763, 32627, 32587, 32856] avg=32768.0000 max=33005 max_rank=1 min=32587 min_rank=6 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[32422, 32697, 32711, 32915, 32901, 32866, 32682, 32950] avg=32768.0000 max=32950 max_rank=7 min=32422 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[32878, 32507, 32462, 32852, 32819, 32795, 32908, 32923] avg=32768.0000 max=32923 max_rank=7 min=32462 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32625, 32739, 32893, 32897, 32729, 32819, 32737, 32705] avg=32768.0000 max=32897 max_rank=3 min=32625 min_rank=0 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32749, 32654, 32588, 32692, 32495, 32911, 32965, 33090] avg=32768.0000 max=33090 max_rank=7 min=32495 min_rank=4 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[32869, 32419, 32835, 32460, 32931, 32891, 32729, 33010] avg=32768.0000 max=33010 max_rank=7 min=32419 min_rank=1 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32411, 32936, 33237, 32808, 32634, 32580] avg=32768.0000 max=33237 max_rank=4 min=32411 min_rank=2 imbalance_ratio=1.014313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32411, 32936, 33237, 32808, 32634, 32580] avg=32768.0000 max=33237 max_rank=4 min=32411 min_rank=2 imbalance_ratio=1.014313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32411, 32936, 33237, 32808, 32634, 32580] avg=32768.0000 max=33237 max_rank=4 min=32411 min_rank=2 imbalance_ratio=1.014313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[32894, 32580, 32492, 32900, 32828, 32901, 33078, 32471] avg=32768.0000 max=33078 max_rank=6 min=32471 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] avg=32768.0000 max=33096 max_rank=4 min=32552 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=141 max_load_reduction_pct=0.004242 moved_to_replicas=141 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32411, 32936, 33237, 32808, 32634, 32580] avg=32768.0000 max=33237 max_rank=4 min=32411 min_rank=2 imbalance_ratio=1.014313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] avg=32768.0000 max=33096 max_rank=4 min=32552 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=141 max_load_reduction_pct=0.004242 moved_to_replicas=141 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32411, 32936, 33237, 32808, 32634, 32580] avg=32768.0000 max=33237 max_rank=4 min=32411 min_rank=2 imbalance_ratio=1.014313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] avg=32768.0000 max=33096 max_rank=4 min=32552 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=141 max_load_reduction_pct=0.004242 moved_to_replicas=141 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] avg=32768.0000 max=33096 max_rank=4 min=32552 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=141 max_load_reduction_pct=0.004242 moved_to_replicas=141 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] avg=32768.0000 max=33096 max_rank=4 min=32552 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=141 max_load_reduction_pct=0.004242 moved_to_replicas=141 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[32681, 32742, 32557, 32872, 32981, 32793, 32823, 32695] avg=32768.0000 max=32981 max_rank=4 min=32557 min_rank=2 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32411, 32936, 33237, 32808, 32634, 32580] avg=32768.0000 max=33237 max_rank=4 min=32411 min_rank=2 imbalance_ratio=1.014313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] avg=32768.0000 max=33096 max_rank=4 min=32552 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=141 max_load_reduction_pct=0.004242 moved_to_replicas=141 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32411, 32936, 33237, 32808, 32634, 32580] avg=32768.0000 max=33237 max_rank=4 min=32411 min_rank=2 imbalance_ratio=1.014313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] avg=32768.0000 max=33096 max_rank=4 min=32552 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=141 max_load_reduction_pct=0.004242 moved_to_replicas=141 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[32991, 32724, 32942, 32625, 32428, 33052, 32647, 32735] avg=32768.0000 max=33052 max_rank=5 min=32428 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32411, 32936, 33237, 32808, 32634, 32580] avg=32768.0000 max=33237 max_rank=4 min=32411 min_rank=2 imbalance_ratio=1.014313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[32730, 32808, 32552, 32936, 33096, 32808, 32634, 32580] avg=32768.0000 max=33096 max_rank=4 min=32552 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=141 max_load_reduction_pct=0.004242 moved_to_replicas=141 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[32942, 32927, 32600, 32895, 32641, 32627, 32924, 32588] avg=32768.0000 max=32942 max_rank=0 min=32588 min_rank=7 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[32498, 32705, 32838, 32876, 32607, 33049, 32833, 32738] avg=32768.0000 max=33049 max_rank=5 min=32498 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[32813, 32943, 32852, 32569, 32602, 32843, 32686, 32836] avg=32768.0000 max=32943 max_rank=1 min=32569 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[32541, 33023, 32884, 32646, 32639, 32827, 32721, 32863] avg=32768.0000 max=33023 max_rank=1 min=32541 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=25 tokens_per_ep_rank=[32588, 32701, 32811, 32759, 32896, 32654, 32785, 32950] avg=32768.0000 max=32950 max_rank=7 min=32588 min_rank=0 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=26 tokens_per_ep_rank=[32733, 32592, 32965, 32867, 32609, 32768, 32716, 32894] avg=32768.0000 max=32965 max_rank=2 min=32592 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=27 tokens_per_ep_rank=[32848, 32900, 32836, 32814, 32713, 32712, 32857, 32464] avg=32768.0000 max=32900 max_rank=1 min=32464 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33101, 32555] avg=32768.0000 max=33101 max_rank=6 min=32555 min_rank=7 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33101, 32555] avg=32768.0000 max=33101 max_rank=6 min=32555 min_rank=7 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33101, 32555] avg=32768.0000 max=33101 max_rank=6 min=32555 min_rank=7 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] avg=32768.0000 max=33096 max_rank=6 min=32560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=5 max_load_reduction_pct=0.000151 moved_to_replicas=5 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33101, 32555] avg=32768.0000 max=33101 max_rank=6 min=32555 min_rank=7 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] avg=32768.0000 max=33096 max_rank=6 min=32560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=5 max_load_reduction_pct=0.000151 moved_to_replicas=5 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33101, 32555] avg=32768.0000 max=33101 max_rank=6 min=32555 min_rank=7 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] avg=32768.0000 max=33096 max_rank=6 min=32560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=5 max_load_reduction_pct=0.000151 moved_to_replicas=5 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=28 tokens_per_ep_rank=[32891, 32796, 32991, 32754, 32524, 32791, 32528, 32869] avg=32768.0000 max=32991 max_rank=2 min=32524 min_rank=4 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] avg=32768.0000 max=33096 max_rank=6 min=32560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=5 max_load_reduction_pct=0.000151 moved_to_replicas=5 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] avg=32768.0000 max=33096 max_rank=6 min=32560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=5 max_load_reduction_pct=0.000151 moved_to_replicas=5 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33101, 32555] avg=32768.0000 max=33101 max_rank=6 min=32555 min_rank=7 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=29 tokens_per_ep_rank=[32740, 33029, 32846, 32588, 32681, 32793, 32843, 32624] avg=32768.0000 max=33029 max_rank=1 min=32588 min_rank=3 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] avg=32768.0000 max=33096 max_rank=6 min=32560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=5 max_load_reduction_pct=0.000151 moved_to_replicas=5 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33101, 32555] avg=32768.0000 max=33101 max_rank=6 min=32555 min_rank=7 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=30 tokens_per_ep_rank=[32224, 32894, 32799, 32997, 33024, 32686, 32838, 32682] avg=32768.0000 max=33024 max_rank=4 min=32224 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] avg=32768.0000 max=33096 max_rank=6 min=32560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=5 max_load_reduction_pct=0.000151 moved_to_replicas=5 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33101, 32555] avg=32768.0000 max=33101 max_rank=6 min=32555 min_rank=7 imbalance_ratio=1.010162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=31 tokens_per_ep_rank=[32756, 32787, 32826, 32707, 32610, 32802, 33096, 32560] avg=32768.0000 max=33096 max_rank=6 min=32560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=5 max_load_reduction_pct=0.000151 moved_to_replicas=5 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33110, 32889, 32682, 32461, 32889] avg=32768.0000 max=33110 max_rank=3 min=32461 min_rank=6 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33110, 32889, 32682, 32461, 32889] avg=32768.0000 max=33110 max_rank=3 min=32461 min_rank=6 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33110, 32889, 32682, 32461, 32889] avg=32768.0000 max=33110 max_rank=3 min=32461 min_rank=6 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33110, 32889, 32682, 32461, 32889] avg=32768.0000 max=33110 max_rank=3 min=32461 min_rank=6 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33110, 32889, 32682, 32461, 32889] avg=32768.0000 max=33110 max_rank=3 min=32461 min_rank=6 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=32 tokens_per_ep_rank=[32597, 32683, 32771, 32887, 32820, 32641, 32774, 32971] avg=32768.0000 max=32971 max_rank=7 min=32597 min_rank=0 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32453, 32541, 32996, 32896, 32576, 32913, 33102] avg=32768.0000 max=33102 max_rank=7 min=32453 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32453, 32541, 32996, 32896, 32576, 32913, 33102] avg=32768.0000 max=33102 max_rank=7 min=32453 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32453, 32541, 32996, 32896, 32576, 32913, 33102] avg=32768.0000 max=33102 max_rank=7 min=32453 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] avg=32768.0000 max=33096 max_rank=7 min=32459 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] avg=32768.0000 max=33096 max_rank=7 min=32459 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33110, 32889, 32682, 32461, 32889] avg=32768.0000 max=33110 max_rank=3 min=32461 min_rank=6 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32453, 32541, 32996, 32896, 32576, 32913, 33102] avg=32768.0000 max=33102 max_rank=7 min=32453 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32453, 32541, 32996, 32896, 32576, 32913, 33102] avg=32768.0000 max=33102 max_rank=7 min=32453 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] avg=32768.0000 max=33096 max_rank=7 min=32459 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=33 tokens_per_ep_rank=[32616, 32947, 32591, 32776, 32794, 32647, 32899, 32874] avg=32768.0000 max=32947 max_rank=1 min=32591 min_rank=2 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] avg=32768.0000 max=33096 max_rank=7 min=32459 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] avg=32768.0000 max=33096 max_rank=7 min=32459 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32453, 32541, 32996, 32896, 32576, 32913, 33102] avg=32768.0000 max=33102 max_rank=7 min=32453 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33110, 32889, 32682, 32461, 32889] avg=32768.0000 max=33110 max_rank=3 min=32461 min_rank=6 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] avg=32768.0000 max=33096 max_rank=7 min=32459 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=34 tokens_per_ep_rank=[32745, 32764, 32656, 32700, 32676, 32969, 32821, 32813] avg=32768.0000 max=32969 max_rank=5 min=32656 min_rank=2 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33110, 32889, 32682, 32461, 32889] avg=32768.0000 max=33110 max_rank=3 min=32461 min_rank=6 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32453, 32541, 32996, 32896, 32576, 32913, 33102] avg=32768.0000 max=33102 max_rank=7 min=32453 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=35 tokens_per_ep_rank=[32742, 32527, 32844, 33096, 32889, 32682, 32475, 32889] avg=32768.0000 max=33096 max_rank=3 min=32475 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] avg=32768.0000 max=33096 max_rank=7 min=32459 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32453, 32541, 32996, 32896, 32576, 32913, 33102] avg=32768.0000 max=33102 max_rank=7 min=32453 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=36 tokens_per_ep_rank=[32667, 32459, 32541, 32996, 32896, 32576, 32913, 33096] avg=32768.0000 max=33096 max_rank=7 min=32459 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=6 max_load_reduction_pct=0.000181 moved_to_replicas=6 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33230, 32825, 32377] avg=32768.0000 max=33230 max_rank=5 min=32377 min_rank=7 imbalance_ratio=1.014099 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33230, 32825, 32377] avg=32768.0000 max=33230 max_rank=5 min=32377 min_rank=7 imbalance_ratio=1.014099 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33230, 32825, 32377] avg=32768.0000 max=33230 max_rank=5 min=32377 min_rank=7 imbalance_ratio=1.014099 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=37 tokens_per_ep_rank=[32567, 32911, 32906, 32505, 32785, 32846, 32672, 32952] avg=32768.0000 max=32952 max_rank=7 min=32505 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] avg=32768.0000 max=33096 max_rank=5 min=32425 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=134 max_load_reduction_pct=0.004033 moved_to_replicas=134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] avg=32768.0000 max=33096 max_rank=5 min=32425 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=134 max_load_reduction_pct=0.004033 moved_to_replicas=134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] avg=32768.0000 max=33096 max_rank=5 min=32425 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=134 max_load_reduction_pct=0.004033 moved_to_replicas=134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33230, 32825, 32377] avg=32768.0000 max=33230 max_rank=5 min=32377 min_rank=7 imbalance_ratio=1.014099 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33230, 32825, 32377] avg=32768.0000 max=33230 max_rank=5 min=32377 min_rank=7 imbalance_ratio=1.014099 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] avg=32768.0000 max=33096 max_rank=5 min=32425 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=134 max_load_reduction_pct=0.004033 moved_to_replicas=134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] avg=32768.0000 max=33096 max_rank=5 min=32425 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=134 max_load_reduction_pct=0.004033 moved_to_replicas=134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=38 tokens_per_ep_rank=[32862, 32788, 32687, 32826, 32700, 32563, 32916, 32802] avg=32768.0000 max=32916 max_rank=6 min=32563 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33230, 32825, 32377] avg=32768.0000 max=33230 max_rank=5 min=32377 min_rank=7 imbalance_ratio=1.014099 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=43 tokens_per_ep_rank=[32400, 33192, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33192 max_rank=1 min=32400 min_rank=0 imbalance_ratio=1.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=43 tokens_per_ep_rank=[32400, 33192, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33192 max_rank=1 min=32400 min_rank=0 imbalance_ratio=1.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=43 tokens_per_ep_rank=[32400, 33192, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33192 max_rank=1 min=32400 min_rank=0 imbalance_ratio=1.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] avg=32768.0000 max=33096 max_rank=5 min=32425 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=134 max_load_reduction_pct=0.004033 moved_to_replicas=134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=43 tokens_per_ep_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33096 max_rank=1 min=32496 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96 max_load_reduction_pct=0.002892 moved_to_replicas=96 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=39 tokens_per_ep_rank=[32925, 32577, 32672, 32787, 32962, 32576, 32943, 32702] avg=32768.0000 max=32962 max_rank=4 min=32576 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=43 tokens_per_ep_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33096 max_rank=1 min=32496 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96 max_load_reduction_pct=0.002892 moved_to_replicas=96 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=43 tokens_per_ep_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33096 max_rank=1 min=32496 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96 max_load_reduction_pct=0.002892 moved_to_replicas=96 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=43 tokens_per_ep_rank=[32400, 33192, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33192 max_rank=1 min=32400 min_rank=0 imbalance_ratio=1.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=43 tokens_per_ep_rank=[32400, 33192, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33192 max_rank=1 min=32400 min_rank=0 imbalance_ratio=1.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33230, 32825, 32377] avg=32768.0000 max=33230 max_rank=5 min=32377 min_rank=7 imbalance_ratio=1.014099 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=43 tokens_per_ep_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33096 max_rank=1 min=32496 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96 max_load_reduction_pct=0.002892 moved_to_replicas=96 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=43 tokens_per_ep_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33096 max_rank=1 min=32496 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96 max_load_reduction_pct=0.002892 moved_to_replicas=96 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] avg=32768.0000 max=33096 max_rank=5 min=32425 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=134 max_load_reduction_pct=0.004033 moved_to_replicas=134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=40 tokens_per_ep_rank=[32676, 32733, 32790, 32776, 32845, 32674, 32823, 32827] avg=32768.0000 max=32845 max_rank=4 min=32674 min_rank=5 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=43 tokens_per_ep_rank=[32400, 33192, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33192 max_rank=1 min=32400 min_rank=0 imbalance_ratio=1.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33114, 33058, 32451, 32913, 32721, 32502] avg=32768.0000 max=33114 max_rank=2 min=32451 min_rank=4 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33114, 33058, 32451, 32913, 32721, 32502] avg=32768.0000 max=33114 max_rank=2 min=32451 min_rank=4 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33114, 33058, 32451, 32913, 32721, 32502] avg=32768.0000 max=33114 max_rank=2 min=32451 min_rank=4 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33230, 32825, 32377] avg=32768.0000 max=33230 max_rank=5 min=32377 min_rank=7 imbalance_ratio=1.014099 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=43 tokens_per_ep_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33096 max_rank=1 min=32496 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96 max_load_reduction_pct=0.002892 moved_to_replicas=96 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] avg=32768.0000 max=33096 max_rank=2 min=32469 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000544 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] avg=32768.0000 max=33096 max_rank=2 min=32469 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000544 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] avg=32768.0000 max=33096 max_rank=2 min=32469 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000544 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=41 tokens_per_ep_rank=[32425, 32729, 32689, 32942, 32927, 33096, 32825, 32511] avg=32768.0000 max=33096 max_rank=5 min=32425 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=134 max_load_reduction_pct=0.004033 moved_to_replicas=134 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33114, 33058, 32451, 32913, 32721, 32502] avg=32768.0000 max=33114 max_rank=2 min=32451 min_rank=4 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33114, 33058, 32451, 32913, 32721, 32502] avg=32768.0000 max=33114 max_rank=2 min=32451 min_rank=4 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=43 tokens_per_ep_rank=[32400, 33192, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33192 max_rank=1 min=32400 min_rank=0 imbalance_ratio=1.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] avg=32768.0000 max=33096 max_rank=2 min=32469 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000544 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] avg=32768.0000 max=33096 max_rank=2 min=32469 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000544 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=43 tokens_per_ep_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33096 max_rank=1 min=32496 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96 max_load_reduction_pct=0.002892 moved_to_replicas=96 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=42 tokens_per_ep_rank=[32944, 32957, 32880, 32834, 32677, 32492, 32859, 32501] avg=32768.0000 max=32957 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33114, 33058, 32451, 32913, 32721, 32502] avg=32768.0000 max=33114 max_rank=2 min=32451 min_rank=4 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] avg=32768.0000 max=33096 max_rank=2 min=32469 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000544 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=43 tokens_per_ep_rank=[32400, 33192, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33192 max_rank=1 min=32400 min_rank=0 imbalance_ratio=1.012939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=43 tokens_per_ep_rank=[32496, 33096, 32892, 32540, 32851, 32872, 32865, 32532] avg=32768.0000 max=33096 max_rank=1 min=32496 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=96 max_load_reduction_pct=0.002892 moved_to_replicas=96 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32480, 32677, 32884, 33023, 33120, 32604, 32650] avg=32768.0000 max=33120 max_rank=5 min=32480 min_rank=1 imbalance_ratio=1.010742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32480, 32677, 32884, 33023, 33120, 32604, 32650] avg=32768.0000 max=33120 max_rank=5 min=32480 min_rank=1 imbalance_ratio=1.010742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33114, 33058, 32451, 32913, 32721, 32502] avg=32768.0000 max=33114 max_rank=2 min=32451 min_rank=4 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32480, 32677, 32884, 33023, 33120, 32604, 32650] avg=32768.0000 max=33120 max_rank=5 min=32480 min_rank=1 imbalance_ratio=1.010742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] avg=32768.0000 max=33096 max_rank=5 min=32504 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=24 max_load_reduction_pct=0.000725 moved_to_replicas=24 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] avg=32768.0000 max=33096 max_rank=5 min=32504 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=24 max_load_reduction_pct=0.000725 moved_to_replicas=24 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] avg=32768.0000 max=33096 max_rank=2 min=32469 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000544 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] avg=32768.0000 max=33096 max_rank=5 min=32504 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=24 max_load_reduction_pct=0.000725 moved_to_replicas=24 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32480, 32677, 32884, 33023, 33120, 32604, 32650] avg=32768.0000 max=33120 max_rank=5 min=32480 min_rank=1 imbalance_ratio=1.010742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=44 tokens_per_ep_rank=[32764, 32773, 32825, 32784, 32963, 32860, 32513, 32662] avg=32768.0000 max=32963 max_rank=4 min=32513 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32480, 32677, 32884, 33023, 33120, 32604, 32650] avg=32768.0000 max=33120 max_rank=5 min=32480 min_rank=1 imbalance_ratio=1.010742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] avg=32768.0000 max=33096 max_rank=5 min=32504 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=24 max_load_reduction_pct=0.000725 moved_to_replicas=24 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] avg=32768.0000 max=33096 max_rank=5 min=32504 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=24 max_load_reduction_pct=0.000725 moved_to_replicas=24 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33114, 33058, 32451, 32913, 32721, 32502] avg=32768.0000 max=33114 max_rank=2 min=32451 min_rank=4 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=45 tokens_per_ep_rank=[32729, 32656, 33096, 33058, 32469, 32913, 32721, 32502] avg=32768.0000 max=33096 max_rank=2 min=32469 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000544 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32480, 32677, 32884, 33023, 33120, 32604, 32650] avg=32768.0000 max=33120 max_rank=5 min=32480 min_rank=1 imbalance_ratio=1.010742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] avg=32768.0000 max=33096 max_rank=5 min=32504 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=24 max_load_reduction_pct=0.000725 moved_to_replicas=24 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=46 tokens_per_ep_rank=[32587, 32951, 32894, 32685, 32469, 32778, 32848, 32932] avg=32768.0000 max=32951 max_rank=1 min=32469 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32480, 32677, 32884, 33023, 33120, 32604, 32650] avg=32768.0000 max=33120 max_rank=5 min=32480 min_rank=1 imbalance_ratio=1.010742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] avg=32768.0000 max=33096 max_rank=5 min=32504 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=24 max_load_reduction_pct=0.000725 moved_to_replicas=24 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=47 tokens_per_ep_rank=[32647, 32805, 32903, 32821, 32568, 32745, 32924, 32731] avg=32768.0000 max=32924 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32480, 32677, 32884, 33023, 33120, 32604, 32650] avg=32768.0000 max=33120 max_rank=5 min=32480 min_rank=1 imbalance_ratio=1.010742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=48 tokens_per_ep_rank=[32706, 32504, 32677, 32884, 33023, 33096, 32604, 32650] avg=32768.0000 max=33096 max_rank=5 min=32504 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=24 max_load_reduction_pct=0.000725 moved_to_replicas=24 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=48 total_expected_recv=1572504 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344576 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=48 total_expected_recv=1573989 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342912 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=48 total_expected_recv=1572398 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343552 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=48 total_expected_recv=1573625 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343296 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=48 total_expected_recv=1572358 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344704 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=48 total_expected_recv=1572700 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344448 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=48 total_expected_recv=1570636 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=348160 peak_demand=32999 required_capacity=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=48 total_expected_recv=1574702 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341888 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:48:17] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 710.1 | throughput per GPU (TFLOP/s/GPU): 125.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.599195E+00 | loss scale: 1.0 | grad norm: 8.736 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33033 expected_overflow=0 exact_recv_per_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32337 expected_overflow=0 exact_recv_per_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=33033 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32337 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33017 expected_overflow=0 exact_recv_per_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33017 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32514 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33069 expected_overflow=0 exact_recv_per_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33069 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33082 expected_overflow=0 exact_recv_per_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=33082 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32495 expected_overflow=0 exact_recv_per_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32495 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33084 expected_overflow=0 exact_recv_per_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32514 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=40960 expected_recv=33084 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32595 expected_overflow=0 exact_recv_per_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32353 expected_overflow=0 exact_recv_per_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32306 expected_overflow=0 exact_recv_per_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32595 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32353 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33033 expected_overflow=0 exact_recv_per_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32306 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33033 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33091 expected_overflow=0 exact_recv_per_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=40960 expected_recv=33091 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32289 expected_overflow=0 exact_recv_per_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32987 expected_overflow=0 exact_recv_per_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=40960 expected_recv=32289 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32987 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32507 expected_overflow=0 exact_recv_per_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32507 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=23 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32945 expected_overflow=0 exact_recv_per_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32945 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33000 expected_overflow=0 exact_recv_per_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32390 expected_overflow=0 exact_recv_per_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=25 calls=1 budget=40960 expected_recv=33000 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32390 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=24 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=25 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=26 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=27 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=28 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=29 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33093 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33068 expected_overflow=0 exact_recv_per_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=30 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=31 calls=1 budget=40960 expected_recv=33068 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33019 expected_overflow=0 exact_recv_per_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=30 calls=1 budget=40960 expected_recv=33019 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33013 expected_overflow=0 exact_recv_per_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33013 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=32 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=33 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32536 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=31 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32522 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=34 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=32 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32983 expected_overflow=0 exact_recv_per_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32983 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32499 expected_overflow=0 exact_recv_per_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=33 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32499 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33071 expected_overflow=0 exact_recv_per_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33071 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=36 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32373 expected_overflow=0 exact_recv_per_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32373 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=34 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33056 expected_overflow=0 exact_recv_per_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=37 calls=1 budget=40960 expected_recv=33056 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32595 expected_overflow=0 exact_recv_per_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32521 expected_overflow=0 exact_recv_per_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=35 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32595 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32521 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32412 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=36 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=38 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=38 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=37 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32938 expected_overflow=0 exact_recv_per_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=38 calls=1 budget=40960 expected_recv=32938 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=39 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32969 expected_overflow=0 exact_recv_per_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32969 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=40 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32291 expected_overflow=0 exact_recv_per_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32291 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32945 expected_overflow=0 exact_recv_per_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32945 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32521 expected_overflow=0 exact_recv_per_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=41 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32430 expected_overflow=0 exact_recv_per_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32521 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32430 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32910 expected_overflow=0 exact_recv_per_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=42 calls=1 budget=40960 expected_recv=32910 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=43 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33079 expected_overflow=0 exact_recv_per_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=43 calls=1 budget=40960 expected_recv=33079 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=44 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32473 expected_overflow=0 exact_recv_per_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32473 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=45 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33109, 33033, 32700, 32324, 32582, 32693, 32610, 33093] avg=32768.0000 max=33109 max_rank=0 min=32324 min_rank=3 imbalance_ratio=1.010406 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=46 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] avg=32768.0000 max=33096 max_rank=0 min=32337 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13 max_load_reduction_pct=0.000393 moved_to_replicas=13 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33017 expected_overflow=0 exact_recv_per_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33109, 33033, 32700, 32324, 32582, 32693, 32610, 33093] avg=32768.0000 max=33109 max_rank=0 min=32324 min_rank=3 imbalance_ratio=1.010406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=47 calls=1 budget=40960 expected_recv=33017 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33109, 33033, 32700, 32324, 32582, 32693, 32610, 33093] avg=32768.0000 max=33109 max_rank=0 min=32324 min_rank=3 imbalance_ratio=1.010406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33109, 33033, 32700, 32324, 32582, 32693, 32610, 33093] avg=32768.0000 max=33109 max_rank=0 min=32324 min_rank=3 imbalance_ratio=1.010406 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33109, 33033, 32700, 32324, 32582, 32693, 32610, 33093] avg=32768.0000 max=33109 max_rank=0 min=32324 min_rank=3 imbalance_ratio=1.010406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] avg=32768.0000 max=33096 max_rank=0 min=32337 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13 max_load_reduction_pct=0.000393 moved_to_replicas=13 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] avg=32768.0000 max=33096 max_rank=0 min=32337 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13 max_load_reduction_pct=0.000393 moved_to_replicas=13 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] avg=32768.0000 max=33096 max_rank=0 min=32337 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13 max_load_reduction_pct=0.000393 moved_to_replicas=13 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=47 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] avg=32768.0000 max=33096 max_rank=0 min=32337 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13 max_load_reduction_pct=0.000393 moved_to_replicas=13 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33109, 33033, 32700, 32324, 32582, 32693, 32610, 33093] avg=32768.0000 max=33109 max_rank=0 min=32324 min_rank=3 imbalance_ratio=1.010406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] avg=32768.0000 max=33096 max_rank=0 min=32337 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13 max_load_reduction_pct=0.000393 moved_to_replicas=13 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[32641, 32803, 32812, 32783, 32701, 32826, 32880, 32698] avg=32768.0000 max=32880 max_rank=6 min=32641 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33109, 33033, 32700, 32324, 32582, 32693, 32610, 33093] avg=32768.0000 max=33109 max_rank=0 min=32324 min_rank=3 imbalance_ratio=1.010406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] avg=32768.0000 max=33096 max_rank=0 min=32337 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13 max_load_reduction_pct=0.000393 moved_to_replicas=13 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33109, 33033, 32700, 32324, 32582, 32693, 32610, 33093] avg=32768.0000 max=33109 max_rank=0 min=32324 min_rank=3 imbalance_ratio=1.010406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33033, 32700, 32337, 32582, 32693, 32610, 33093] avg=32768.0000 max=33096 max_rank=0 min=32337 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13 max_load_reduction_pct=0.000393 moved_to_replicas=13 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[33028, 32993, 32624, 32607, 32804, 32486, 32834, 32768] avg=32768.0000 max=33028 max_rank=0 min=32486 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32700, 32934, 33017, 32823, 32806, 32647, 32636, 32581] avg=32768.0000 max=33017 max_rank=2 min=32581 min_rank=7 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32921, 32875, 32971, 32806, 32711, 32505, 32542, 32813] avg=32768.0000 max=32971 max_rank=2 min=32505 min_rank=5 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32575, 32925, 32924, 32813, 32821, 32635, 32514, 32937] avg=32768.0000 max=32937 max_rank=7 min=32514 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32794, 32820, 32828, 32780, 32763, 32401, 32899, 32859] avg=32768.0000 max=32899 max_rank=6 min=32401 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32909, 32924, 32797, 32656, 32719, 33040, 32525, 32574] avg=32768.0000 max=33040 max_rank=5 min=32525 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32722, 32756, 33069, 32656, 33016, 32606, 32759, 32560] avg=32768.0000 max=33069 max_rank=2 min=32560 min_rank=7 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32697, 32815, 32922, 32538, 32706, 33082, 32677, 32707] avg=32768.0000 max=33082 max_rank=5 min=32538 min_rank=3 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32831, 32884, 32988, 32921, 32478, 32610, 32686, 32746] avg=32768.0000 max=32988 max_rank=2 min=32478 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32842, 33065, 32495, 32617, 32836, 32567, 32922, 32800] avg=32768.0000 max=33065 max_rank=1 min=32495 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32739, 32645, 32879, 32929, 32768, 32634, 32829, 32721] avg=32768.0000 max=32929 max_rank=3 min=32634 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32905, 32765, 32908, 32643, 32475, 32780, 32900, 32768] avg=32768.0000 max=32908 max_rank=2 min=32475 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32353, 33084, 32630, 32840, 33093, 32514, 32780, 32850] avg=32768.0000 max=33093 max_rank=4 min=32353 min_rank=0 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32837, 32630, 32595, 33091, 33033, 32895, 32306, 32757] avg=32768.0000 max=33091 max_rank=3 min=32306 min_rank=6 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[32827, 32853, 32777, 32663, 32827, 32567, 32773, 32857] avg=32768.0000 max=32857 max_rank=7 min=32567 min_rank=5 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[32845, 32864, 32779, 32289, 32884, 32852, 32688, 32943] avg=32768.0000 max=32943 max_rank=7 min=32289 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[32728, 32867, 33018, 32695, 32575, 32840, 32902, 32519] avg=32768.0000 max=33018 max_rank=2 min=32519 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[33110, 32784, 32558, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33110 max_rank=0 min=32558 min_rank=2 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[33110, 32784, 32558, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33110 max_rank=0 min=32558 min_rank=2 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33096 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33096 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[33110, 32784, 32558, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33110 max_rank=0 min=32558 min_rank=2 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[33035, 32568, 32655, 32956, 32512, 32612, 32819, 32987] avg=32768.0000 max=33035 max_rank=0 min=32512 min_rank=4 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[33110, 32784, 32558, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33110 max_rank=0 min=32558 min_rank=2 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33096 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[33110, 32784, 32558, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33110 max_rank=0 min=32558 min_rank=2 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33096 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33096 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[32592, 32827, 32789, 32688, 32894, 32872, 32713, 32769] avg=32768.0000 max=32894 max_rank=4 min=32592 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[33110, 32784, 32558, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33110 max_rank=0 min=32558 min_rank=2 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33096 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[32898, 32808, 32897, 32884, 32654, 32770, 32507, 32726] avg=32768.0000 max=32898 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33103, 32383] avg=32768.0000 max=33103 max_rank=6 min=32383 min_rank=7 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33103, 32383] avg=32768.0000 max=33103 max_rank=6 min=32383 min_rank=7 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] avg=32768.0000 max=33096 max_rank=6 min=32390 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] avg=32768.0000 max=33096 max_rank=6 min=32390 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33103, 32383] avg=32768.0000 max=33103 max_rank=6 min=32383 min_rank=7 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33103, 32383] avg=32768.0000 max=33103 max_rank=6 min=32383 min_rank=7 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[32745, 32874, 32769, 32743, 32699, 32677, 32892, 32745] avg=32768.0000 max=32892 max_rank=6 min=32677 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] avg=32768.0000 max=33096 max_rank=6 min=32390 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[33110, 32784, 32558, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33110 max_rank=0 min=32558 min_rank=2 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] avg=32768.0000 max=33096 max_rank=6 min=32390 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33103, 32383] avg=32768.0000 max=33103 max_rank=6 min=32383 min_rank=7 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33096 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[33110, 32784, 32558, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33110 max_rank=0 min=32558 min_rank=2 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] avg=32768.0000 max=33096 max_rank=6 min=32390 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[33096, 32784, 32572, 32696, 32754, 32719, 32672, 32851] avg=32768.0000 max=33096 max_rank=0 min=32572 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33103, 32383] avg=32768.0000 max=33103 max_rank=6 min=32383 min_rank=7 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] avg=32768.0000 max=33096 max_rank=6 min=32390 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=30 tokens_per_ep_rank=[33163, 32889, 32701, 32599, 32768, 32682, 32693, 32649] avg=32768.0000 max=33163 max_rank=0 min=32599 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=25 tokens_per_ep_rank=[32567, 32889, 32808, 32935, 32850, 32591, 32612, 32892] avg=32768.0000 max=32935 max_rank=3 min=32567 min_rank=0 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=30 tokens_per_ep_rank=[33163, 32889, 32701, 32599, 32768, 32682, 32693, 32649] avg=32768.0000 max=33163 max_rank=0 min=32599 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=30 tokens_per_ep_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] avg=32768.0000 max=33096 max_rank=0 min=32649 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=30 tokens_per_ep_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] avg=32768.0000 max=33096 max_rank=0 min=32649 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=30 tokens_per_ep_rank=[33163, 32889, 32701, 32599, 32768, 32682, 32693, 32649] avg=32768.0000 max=33163 max_rank=0 min=32599 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=30 tokens_per_ep_rank=[33163, 32889, 32701, 32599, 32768, 32682, 32693, 32649] avg=32768.0000 max=33163 max_rank=0 min=32599 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=30 tokens_per_ep_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] avg=32768.0000 max=33096 max_rank=0 min=32649 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=26 tokens_per_ep_rank=[32569, 32606, 32928, 32730, 33000, 32908, 32945, 32458] avg=32768.0000 max=33000 max_rank=4 min=32458 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33103, 32383] avg=32768.0000 max=33103 max_rank=6 min=32383 min_rank=7 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=30 tokens_per_ep_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] avg=32768.0000 max=33096 max_rank=0 min=32649 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=30 tokens_per_ep_rank=[33163, 32889, 32701, 32599, 32768, 32682, 32693, 32649] avg=32768.0000 max=33163 max_rank=0 min=32599 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] avg=32768.0000 max=33096 max_rank=6 min=32390 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33103, 32383] avg=32768.0000 max=33103 max_rank=6 min=32383 min_rank=7 imbalance_ratio=1.010223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=30 tokens_per_ep_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] avg=32768.0000 max=33096 max_rank=0 min=32649 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=27 tokens_per_ep_rank=[32790, 32631, 32855, 32845, 32777, 32760, 33096, 32390] avg=32768.0000 max=33096 max_rank=6 min=32390 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=7 max_load_reduction_pct=0.000211 moved_to_replicas=7 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=30 tokens_per_ep_rank=[33163, 32889, 32701, 32599, 32768, 32682, 32693, 32649] avg=32768.0000 max=33163 max_rank=0 min=32599 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=30 tokens_per_ep_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] avg=32768.0000 max=33096 max_rank=0 min=32649 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=28 tokens_per_ep_rank=[32784, 32867, 32847, 32600, 32919, 32826, 32820, 32481] avg=32768.0000 max=32919 max_rank=4 min=32481 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=29 tokens_per_ep_rank=[32782, 32729, 32800, 32716, 32558, 32771, 32855, 32933] avg=32768.0000 max=32933 max_rank=7 min=32558 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=30 tokens_per_ep_rank=[33163, 32889, 32701, 32599, 32768, 32682, 32693, 32649] avg=32768.0000 max=33163 max_rank=0 min=32599 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=30 tokens_per_ep_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] avg=32768.0000 max=33096 max_rank=0 min=32649 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=30 tokens_per_ep_rank=[33163, 32889, 32701, 32599, 32768, 32682, 32693, 32649] avg=32768.0000 max=33163 max_rank=0 min=32599 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=30 tokens_per_ep_rank=[33096, 32889, 32701, 32666, 32768, 32682, 32693, 32649] avg=32768.0000 max=33096 max_rank=0 min=32649 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=31 tokens_per_ep_rank=[32694, 32749, 32659, 33019, 32777, 32850, 32673, 32723] avg=32768.0000 max=33019 max_rank=3 min=32659 min_rank=2 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=32 tokens_per_ep_rank=[32705, 33068, 32535, 32533, 32712, 32918, 33093, 32580] avg=32768.0000 max=33093 max_rank=6 min=32533 min_rank=3 imbalance_ratio=1.009918 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=33 tokens_per_ep_rank=[33025, 32597, 32549, 33006, 32917, 32823, 32558, 32669] avg=32768.0000 max=33025 max_rank=0 min=32549 min_rank=2 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=34 tokens_per_ep_rank=[32983, 32882, 33005, 32640, 32522, 32563, 32536, 33013] avg=32768.0000 max=33013 max_rank=7 min=32522 min_rank=4 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32399, 32502, 33201, 32848, 32779, 32720] avg=32768.0000 max=33201 max_rank=4 min=32399 min_rank=2 imbalance_ratio=1.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32399, 32502, 33201, 32848, 32779, 32720] avg=32768.0000 max=33201 max_rank=4 min=32399 min_rank=2 imbalance_ratio=1.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=105 max_load_reduction_pct=0.003163 moved_to_replicas=105 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=35 tokens_per_ep_rank=[32552, 32686, 32796, 32762, 32627, 33065, 32832, 32824] avg=32768.0000 max=33065 max_rank=5 min=32552 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=105 max_load_reduction_pct=0.003163 moved_to_replicas=105 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32399, 32502, 33201, 32848, 32779, 32720] avg=32768.0000 max=33201 max_rank=4 min=32399 min_rank=2 imbalance_ratio=1.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32399, 32502, 33201, 32848, 32779, 32720] avg=32768.0000 max=33201 max_rank=4 min=32399 min_rank=2 imbalance_ratio=1.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=41 tokens_per_ep_rank=[33165, 32826, 32848, 32794, 32676, 32646, 32549, 32640] avg=32768.0000 max=33165 max_rank=0 min=32549 min_rank=6 imbalance_ratio=1.012115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=105 max_load_reduction_pct=0.003163 moved_to_replicas=105 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=41 tokens_per_ep_rank=[33165, 32826, 32848, 32794, 32676, 32646, 32549, 32640] avg=32768.0000 max=33165 max_rank=0 min=32549 min_rank=6 imbalance_ratio=1.012115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=105 max_load_reduction_pct=0.003163 moved_to_replicas=105 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=41 tokens_per_ep_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] avg=32768.0000 max=33096 max_rank=0 min=32618 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=69 max_load_reduction_pct=0.002081 moved_to_replicas=69 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32399, 32502, 33201, 32848, 32779, 32720] avg=32768.0000 max=33201 max_rank=4 min=32399 min_rank=2 imbalance_ratio=1.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=36 tokens_per_ep_rank=[32770, 32869, 32568, 32807, 32818, 32911, 32656, 32745] avg=32768.0000 max=32911 max_rank=5 min=32568 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=41 tokens_per_ep_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] avg=32768.0000 max=33096 max_rank=0 min=32618 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=69 max_load_reduction_pct=0.002081 moved_to_replicas=69 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=41 tokens_per_ep_rank=[33165, 32826, 32848, 32794, 32676, 32646, 32549, 32640] avg=32768.0000 max=33165 max_rank=0 min=32549 min_rank=6 imbalance_ratio=1.012115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=41 tokens_per_ep_rank=[33165, 32826, 32848, 32794, 32676, 32646, 32549, 32640] avg=32768.0000 max=33165 max_rank=0 min=32549 min_rank=6 imbalance_ratio=1.012115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=105 max_load_reduction_pct=0.003163 moved_to_replicas=105 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32566, 32785, 33127, 32639, 32706] avg=32768.0000 max=33127 max_rank=5 min=32566 min_rank=3 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=41 tokens_per_ep_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] avg=32768.0000 max=33096 max_rank=0 min=32618 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=69 max_load_reduction_pct=0.002081 moved_to_replicas=69 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=41 tokens_per_ep_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] avg=32768.0000 max=33096 max_rank=0 min=32618 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=69 max_load_reduction_pct=0.002081 moved_to_replicas=69 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32566, 32785, 33127, 32639, 32706] avg=32768.0000 max=33127 max_rank=5 min=32566 min_rank=3 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=41 tokens_per_ep_rank=[33165, 32826, 32848, 32794, 32676, 32646, 32549, 32640] avg=32768.0000 max=33165 max_rank=0 min=32549 min_rank=6 imbalance_ratio=1.012115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=37 tokens_per_ep_rank=[32521, 32794, 33071, 32980, 32373, 33016, 32890, 32499] avg=32768.0000 max=33071 max_rank=2 min=32373 min_rank=4 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32566, 32785, 33127, 32639, 32706] avg=32768.0000 max=33127 max_rank=5 min=32566 min_rank=3 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32566, 32785, 33127, 32639, 32706] avg=32768.0000 max=33127 max_rank=5 min=32566 min_rank=3 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=41 tokens_per_ep_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] avg=32768.0000 max=33096 max_rank=0 min=32618 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=69 max_load_reduction_pct=0.002081 moved_to_replicas=69 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32399, 32502, 33201, 32848, 32779, 32720] avg=32768.0000 max=33201 max_rank=4 min=32399 min_rank=2 imbalance_ratio=1.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32566, 32785, 33127, 32639, 32706] avg=32768.0000 max=33127 max_rank=5 min=32566 min_rank=3 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=105 max_load_reduction_pct=0.003163 moved_to_replicas=105 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=38 tokens_per_ep_rank=[32600, 32585, 32954, 32811, 33056, 32651, 32569, 32918] avg=32768.0000 max=33056 max_rank=4 min=32569 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=41 tokens_per_ep_rank=[33165, 32826, 32848, 32794, 32676, 32646, 32549, 32640] avg=32768.0000 max=33165 max_rank=0 min=32549 min_rank=6 imbalance_ratio=1.012115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=41 tokens_per_ep_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] avg=32768.0000 max=33096 max_rank=0 min=32618 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=69 max_load_reduction_pct=0.002081 moved_to_replicas=69 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=39 tokens_per_ep_rank=[32923, 33072, 32901, 32938, 32699, 32412, 32604, 32595] avg=32768.0000 max=33072 max_rank=1 min=32412 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32399, 32502, 33201, 32848, 32779, 32720] avg=32768.0000 max=33201 max_rank=4 min=32399 min_rank=2 imbalance_ratio=1.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32566, 32785, 33127, 32639, 32706] avg=32768.0000 max=33127 max_rank=5 min=32566 min_rank=3 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=105 max_load_reduction_pct=0.003163 moved_to_replicas=105 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32399, 32502, 33201, 32848, 32779, 32720] avg=32768.0000 max=33201 max_rank=4 min=32399 min_rank=2 imbalance_ratio=1.013214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=40 tokens_per_ep_rank=[32904, 32791, 32504, 32502, 33096, 32848, 32779, 32720] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=105 max_load_reduction_pct=0.003163 moved_to_replicas=105 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=41 tokens_per_ep_rank=[33165, 32826, 32848, 32794, 32676, 32646, 32549, 32640] avg=32768.0000 max=33165 max_rank=0 min=32549 min_rank=6 imbalance_ratio=1.012115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=41 tokens_per_ep_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] avg=32768.0000 max=33096 max_rank=0 min=32618 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=69 max_load_reduction_pct=0.002081 moved_to_replicas=69 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=41 tokens_per_ep_rank=[33165, 32826, 32848, 32794, 32676, 32646, 32549, 32640] avg=32768.0000 max=33165 max_rank=0 min=32549 min_rank=6 imbalance_ratio=1.012115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=41 tokens_per_ep_rank=[33096, 32826, 32848, 32794, 32676, 32646, 32618, 32640] avg=32768.0000 max=33096 max_rank=0 min=32618 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=69 max_load_reduction_pct=0.002081 moved_to_replicas=69 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32566, 32785, 33127, 32639, 32706] avg=32768.0000 max=33127 max_rank=5 min=32566 min_rank=3 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32432, 32808, 32434, 33155] avg=32768.0000 max=33155 max_rank=7 min=32432 min_rank=4 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32432, 32808, 32434, 33155] avg=32768.0000 max=33155 max_rank=7 min=32432 min_rank=4 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] avg=32768.0000 max=33096 max_rank=7 min=32434 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32566, 32785, 33127, 32639, 32706] avg=32768.0000 max=33127 max_rank=5 min=32566 min_rank=3 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] avg=32768.0000 max=33096 max_rank=7 min=32434 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32432, 32808, 32434, 33155] avg=32768.0000 max=33155 max_rank=7 min=32432 min_rank=4 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32432, 32808, 32434, 33155] avg=32768.0000 max=33155 max_rank=7 min=32432 min_rank=4 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=42 tokens_per_ep_rank=[32658, 32969, 32694, 32597, 32785, 33096, 32639, 32706] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] avg=32768.0000 max=33096 max_rank=7 min=32434 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] avg=32768.0000 max=33096 max_rank=7 min=32434 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32432, 32808, 32434, 33155] avg=32768.0000 max=33155 max_rank=7 min=32432 min_rank=4 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=43 tokens_per_ep_rank=[32521, 32934, 32669, 32910, 32923, 32870, 32588, 32729] avg=32768.0000 max=32934 max_rank=1 min=32521 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] avg=32768.0000 max=33096 max_rank=7 min=32434 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=44 tokens_per_ep_rank=[32771, 32906, 32976, 33079, 32291, 32636, 32867, 32618] avg=32768.0000 max=33079 max_rank=3 min=32291 min_rank=4 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32432, 32808, 32434, 33155] avg=32768.0000 max=33155 max_rank=7 min=32432 min_rank=4 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] avg=32768.0000 max=33096 max_rank=7 min=32434 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=45 tokens_per_ep_rank=[32731, 32956, 32750, 32854, 32561, 32430, 32917, 32945] avg=32768.0000 max=32956 max_rank=1 min=32430 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=46 tokens_per_ep_rank=[32740, 32839, 32697, 32924, 32660, 32656, 32977, 32651] avg=32768.0000 max=32977 max_rank=6 min=32651 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32432, 32808, 32434, 33155] avg=32768.0000 max=33155 max_rank=7 min=32432 min_rank=4 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] avg=32768.0000 max=33096 max_rank=7 min=32434 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32432, 32808, 32434, 33155] avg=32768.0000 max=33155 max_rank=7 min=32432 min_rank=4 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=47 tokens_per_ep_rank=[32871, 32732, 32883, 32829, 32491, 32808, 32434, 33096] avg=32768.0000 max=33096 max_rank=7 min=32434 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=48 tokens_per_ep_rank=[33017, 32706, 32680, 32939, 32669, 32473, 32866, 32794] avg=32768.0000 max=33017 max_rank=0 min=32473 min_rank=5 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=48 total_expected_recv=1571040 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345088 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=48 total_expected_recv=1571897 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344320 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=48 total_expected_recv=1574093 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344192 peak_demand=33071 required_capacity=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=48 total_expected_recv=1571382 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345984 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=48 total_expected_recv=1571636 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=347776 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=48 total_expected_recv=1575968 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341632 peak_demand=33084 required_capacity=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=48 total_expected_recv=1574026 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341760 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=48 total_expected_recv=1572870 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345216 peak_demand=33091 required_capacity=1.009857 + [2026-08-26 15:48:18] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 706.6 | throughput per GPU (TFLOP/s/GPU): 126.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.308316E+00 | loss scale: 1.0 | grad norm: 4.105 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32327 expected_overflow=0 exact_recv_per_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33012 expected_overflow=0 exact_recv_per_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32327 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33012 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33069 expected_overflow=0 exact_recv_per_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=33069 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32553 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32914 expected_overflow=0 exact_recv_per_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32914 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33046 expected_overflow=0 exact_recv_per_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33046 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33054 expected_overflow=0 exact_recv_per_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=33054 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32507 expected_overflow=0 exact_recv_per_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32507 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33048 expected_overflow=0 exact_recv_per_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32355 expected_overflow=0 exact_recv_per_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=40960 expected_recv=33048 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32355 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32331 expected_overflow=0 exact_recv_per_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32331 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33047 expected_overflow=0 exact_recv_per_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=40960 expected_recv=33047 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32914 expected_overflow=0 exact_recv_per_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=40960 expected_recv=32914 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32619 expected_overflow=0 exact_recv_per_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32619 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32554 expected_overflow=0 exact_recv_per_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=40960 expected_recv=32554 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32426 expected_overflow=0 exact_recv_per_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32506 expected_overflow=0 exact_recv_per_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=40960 expected_recv=32426 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32506 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33066 expected_overflow=0 exact_recv_per_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=40960 expected_recv=33066 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32412 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32983 expected_overflow=0 exact_recv_per_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32983 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32492 expected_overflow=0 exact_recv_per_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32492 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32383 expected_overflow=0 exact_recv_per_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32959 expected_overflow=0 exact_recv_per_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32383 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32959 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=23 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=25 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=23 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33008 expected_overflow=0 exact_recv_per_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=26 calls=1 budget=40960 expected_recv=33008 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=24 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=25 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=27 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=26 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=27 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32403 expected_overflow=0 exact_recv_per_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32403 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=28 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33002 expected_overflow=0 exact_recv_per_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=30 calls=1 budget=40960 expected_recv=33002 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=29 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=31 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=30 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32974 expected_overflow=0 exact_recv_per_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32974 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33022 expected_overflow=0 exact_recv_per_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=31 calls=1 budget=40960 expected_recv=33022 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=32 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=35 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32376 expected_overflow=0 exact_recv_per_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=33 calls=1 budget=40960 expected_recv=32376 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=34 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32964 expected_overflow=0 exact_recv_per_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=35 calls=1 budget=40960 expected_recv=32964 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=36 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=38 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=37 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=38 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32490 expected_overflow=0 exact_recv_per_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=40 calls=1 budget=40960 expected_recv=32490 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=39 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33090 expected_overflow=0 exact_recv_per_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32443 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33038 expected_overflow=0 exact_recv_per_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=40 calls=1 budget=40960 expected_recv=33090 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=43 calls=1 budget=40960 expected_recv=33038 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32578 expected_overflow=0 exact_recv_per_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=41 calls=1 budget=40960 expected_recv=32578 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33003 expected_overflow=0 exact_recv_per_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=42 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33003 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33031 expected_overflow=0 exact_recv_per_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=46 calls=1 budget=40960 expected_recv=33031 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32427 expected_overflow=0 exact_recv_per_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32427 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=43 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=45 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=47 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=44 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33019 expected_overflow=0 exact_recv_per_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=45 calls=1 budget=40960 expected_recv=33019 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33158, 32547] avg=32768.0000 max=33158 max_rank=6 min=32547 min_rank=7 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32586 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33158, 32547] avg=32768.0000 max=33158 max_rank=6 min=32547 min_rank=7 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33158, 32547] avg=32768.0000 max=33158 max_rank=6 min=32547 min_rank=7 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=46 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33158, 32547] avg=32768.0000 max=33158 max_rank=6 min=32547 min_rank=7 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32586 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32586 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32627, 32807, 32649, 32651, 32647, 32789, 33138] avg=32768.0000 max=33138 max_rank=7 min=32627 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32586 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33158, 32547] avg=32768.0000 max=33158 max_rank=6 min=32547 min_rank=7 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] avg=32768.0000 max=33096 max_rank=7 min=32647 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32627, 32807, 32649, 32651, 32647, 32789, 33138] avg=32768.0000 max=33138 max_rank=7 min=32627 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=47 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32627, 32807, 32649, 32651, 32647, 32789, 33138] avg=32768.0000 max=33138 max_rank=7 min=32627 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32586 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32627, 32807, 32649, 32651, 32647, 32789, 33138] avg=32768.0000 max=33138 max_rank=7 min=32627 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] avg=32768.0000 max=33096 max_rank=7 min=32647 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33158, 32547] avg=32768.0000 max=33158 max_rank=6 min=32547 min_rank=7 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] avg=32768.0000 max=33096 max_rank=7 min=32647 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] avg=32768.0000 max=33096 max_rank=7 min=32647 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32627, 32807, 32649, 32651, 32647, 32789, 33138] avg=32768.0000 max=33138 max_rank=7 min=32627 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32586 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33158, 32547] avg=32768.0000 max=33158 max_rank=6 min=32547 min_rank=7 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] avg=32768.0000 max=33096 max_rank=7 min=32647 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32586 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32671, 32986, 32703, 33012, 32846, 32710, 32889, 32327] avg=32768.0000 max=33012 max_rank=3 min=32327 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32627, 32807, 32649, 32651, 32647, 32789, 33138] avg=32768.0000 max=33138 max_rank=7 min=32627 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] avg=32768.0000 max=33096 max_rank=7 min=32647 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32627, 32807, 32649, 32651, 32647, 32789, 33138] avg=32768.0000 max=33138 max_rank=7 min=32627 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33158, 32547] avg=32768.0000 max=33158 max_rank=6 min=32547 min_rank=7 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] avg=32768.0000 max=33096 max_rank=7 min=32647 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32685, 32715, 32943, 32824, 32686, 32586, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32586 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32627, 32807, 32649, 32651, 32647, 32789, 33138] avg=32768.0000 max=33138 max_rank=7 min=32627 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32836, 32669, 32807, 32649, 32651, 32647, 32789, 33096] avg=32768.0000 max=33096 max_rank=7 min=32647 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32827, 32528, 32979, 32738, 32767, 32592, 32908, 32805] avg=32768.0000 max=32979 max_rank=2 min=32528 min_rank=1 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32727, 32940, 32925, 32630, 32586, 32798, 32900, 32638] avg=32768.0000 max=32940 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32744, 32835, 32615, 32825, 32591, 32968, 32811, 32755] avg=32768.0000 max=32968 max_rank=5 min=32591 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32900, 32737, 32486, 32553, 32828, 32594, 33069, 32977] avg=32768.0000 max=33069 max_rank=6 min=32486 min_rank=2 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32721, 32653, 32970, 32935, 32839, 32618, 32617, 32791] avg=32768.0000 max=32970 max_rank=2 min=32617 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[33046, 32773, 32788, 32538, 32757, 32914, 32659, 32669] avg=32768.0000 max=33046 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32709, 32821, 32648, 32792, 32871, 32641, 32608, 33054] avg=32768.0000 max=33054 max_rank=7 min=32608 min_rank=6 imbalance_ratio=1.008728 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[33025, 32633, 32769, 32892, 32626, 32724, 32507, 32968] avg=32768.0000 max=33025 max_rank=0 min=32507 min_rank=6 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[32776, 32648, 32355, 32979, 32873, 33048, 32818, 32647] avg=32768.0000 max=33048 max_rank=5 min=32355 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32563, 32608, 32847, 32807, 32655, 32793, 33065, 32806] avg=32768.0000 max=33065 max_rank=6 min=32563 min_rank=0 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32818, 32816, 32331, 32914, 32802, 32730, 32846, 32887] avg=32768.0000 max=32914 max_rank=3 min=32331 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32872, 32732, 32618, 32682, 32568, 32970, 33047, 32655] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=4 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32539, 32881, 32817, 32554, 32886, 32916, 32831, 32720] avg=32768.0000 max=32916 max_rank=5 min=32539 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[32961, 32944, 32619, 32426, 32966, 32717, 32626, 32885] avg=32768.0000 max=32966 max_rank=4 min=32426 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[32754, 32651, 32741, 32801, 32562, 32783, 32920, 32932] avg=32768.0000 max=32932 max_rank=7 min=32562 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[32923, 32918, 32758, 32672, 33005, 32506, 32709, 32653] avg=32768.0000 max=33005 max_rank=4 min=32506 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[32747, 32569, 32412, 32919, 32592, 33066, 32934, 32905] avg=32768.0000 max=33066 max_rank=5 min=32412 min_rank=2 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=26 tokens_per_ep_rank=[32610, 32664, 32632, 32638, 32863, 33111, 32644, 32982] avg=32768.0000 max=33111 max_rank=5 min=32610 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[32857, 32665, 32804, 32747, 32539, 32669, 32933, 32930] avg=32768.0000 max=32933 max_rank=6 min=32539 min_rank=4 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=26 tokens_per_ep_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] avg=32768.0000 max=33096 max_rank=5 min=32625 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=26 tokens_per_ep_rank=[32610, 32664, 32632, 32638, 32863, 33111, 32644, 32982] avg=32768.0000 max=33111 max_rank=5 min=32610 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=26 tokens_per_ep_rank=[32610, 32664, 32632, 32638, 32863, 33111, 32644, 32982] avg=32768.0000 max=33111 max_rank=5 min=32610 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=26 tokens_per_ep_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] avg=32768.0000 max=33096 max_rank=5 min=32625 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=26 tokens_per_ep_rank=[32610, 32664, 32632, 32638, 32863, 33111, 32644, 32982] avg=32768.0000 max=33111 max_rank=5 min=32610 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=26 tokens_per_ep_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] avg=32768.0000 max=33096 max_rank=5 min=32625 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[32792, 32890, 32876, 32624, 32714, 32805, 32983, 32460] avg=32768.0000 max=32983 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=26 tokens_per_ep_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] avg=32768.0000 max=33096 max_rank=5 min=32625 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=26 tokens_per_ep_rank=[32610, 32664, 32632, 32638, 32863, 33111, 32644, 32982] avg=32768.0000 max=33111 max_rank=5 min=32610 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=28 tokens_per_ep_rank=[32534, 32853, 33234, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33234 max_rank=2 min=32534 min_rank=0 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=26 tokens_per_ep_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] avg=32768.0000 max=33096 max_rank=5 min=32625 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[32982, 32383, 32855, 32883, 32775, 32908, 32492, 32866] avg=32768.0000 max=32982 max_rank=0 min=32383 min_rank=1 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=28 tokens_per_ep_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33096 max_rank=2 min=32626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=28 tokens_per_ep_rank=[32534, 32853, 33234, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33234 max_rank=2 min=32534 min_rank=0 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=28 tokens_per_ep_rank=[32534, 32853, 33234, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33234 max_rank=2 min=32534 min_rank=0 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=26 tokens_per_ep_rank=[32610, 32664, 32632, 32638, 32863, 33111, 32644, 32982] avg=32768.0000 max=33111 max_rank=5 min=32610 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=28 tokens_per_ep_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33096 max_rank=2 min=32626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=28 tokens_per_ep_rank=[32534, 32853, 33234, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33234 max_rank=2 min=32534 min_rank=0 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=28 tokens_per_ep_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33096 max_rank=2 min=32626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=26 tokens_per_ep_rank=[32610, 32664, 32632, 32638, 32863, 33111, 32644, 32982] avg=32768.0000 max=33111 max_rank=5 min=32610 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[32837, 33005, 32715, 32739, 32959, 32443, 32832, 32614] avg=32768.0000 max=33005 max_rank=1 min=32443 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=26 tokens_per_ep_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] avg=32768.0000 max=33096 max_rank=5 min=32625 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=28 tokens_per_ep_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33096 max_rank=2 min=32626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=26 tokens_per_ep_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] avg=32768.0000 max=33096 max_rank=5 min=32625 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=28 tokens_per_ep_rank=[32534, 32853, 33234, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33234 max_rank=2 min=32534 min_rank=0 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32505, 32735, 33187, 32674] avg=32768.0000 max=33187 max_rank=6 min=32505 min_rank=4 imbalance_ratio=1.012787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=28 tokens_per_ep_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33096 max_rank=2 min=32626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] avg=32768.0000 max=33096 max_rank=6 min=32596 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=91 max_load_reduction_pct=0.002742 moved_to_replicas=91 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=25 tokens_per_ep_rank=[32794, 32638, 32502, 32889, 32873, 32847, 32655, 32946] avg=32768.0000 max=32946 max_rank=7 min=32502 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32505, 32735, 33187, 32674] avg=32768.0000 max=33187 max_rank=6 min=32505 min_rank=4 imbalance_ratio=1.012787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32505, 32735, 33187, 32674] avg=32768.0000 max=33187 max_rank=6 min=32505 min_rank=4 imbalance_ratio=1.012787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=28 tokens_per_ep_rank=[32534, 32853, 33234, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33234 max_rank=2 min=32534 min_rank=0 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=26 tokens_per_ep_rank=[32610, 32664, 32632, 32638, 32863, 33111, 32644, 32982] avg=32768.0000 max=33111 max_rank=5 min=32610 min_rank=0 imbalance_ratio=1.010468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] avg=32768.0000 max=33096 max_rank=6 min=32596 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=91 max_load_reduction_pct=0.002742 moved_to_replicas=91 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] avg=32768.0000 max=33096 max_rank=6 min=32596 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=91 max_load_reduction_pct=0.002742 moved_to_replicas=91 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32505, 32735, 33187, 32674] avg=32768.0000 max=33187 max_rank=6 min=32505 min_rank=4 imbalance_ratio=1.012787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=28 tokens_per_ep_rank=[32534, 32853, 33234, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33234 max_rank=2 min=32534 min_rank=0 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=28 tokens_per_ep_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33096 max_rank=2 min=32626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=26 tokens_per_ep_rank=[32625, 32664, 32632, 32638, 32863, 33096, 32644, 32982] avg=32768.0000 max=33096 max_rank=5 min=32625 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=15 max_load_reduction_pct=0.000453 moved_to_replicas=15 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] avg=32768.0000 max=33096 max_rank=6 min=32596 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=91 max_load_reduction_pct=0.002742 moved_to_replicas=91 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=28 tokens_per_ep_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33096 max_rank=2 min=32626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32505, 32735, 33187, 32674] avg=32768.0000 max=33187 max_rank=6 min=32505 min_rank=4 imbalance_ratio=1.012787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33220, 32731, 33022, 32470, 32486, 32834, 32577] avg=32768.0000 max=33220 max_rank=1 min=32470 min_rank=4 imbalance_ratio=1.013794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] avg=32768.0000 max=33096 max_rank=6 min=32596 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=91 max_load_reduction_pct=0.002742 moved_to_replicas=91 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] avg=32768.0000 max=33096 max_rank=1 min=32486 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124 max_load_reduction_pct=0.003733 moved_to_replicas=124 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=27 tokens_per_ep_rank=[32907, 32781, 32558, 32558, 32751, 33008, 32724, 32857] avg=32768.0000 max=33008 max_rank=5 min=32558 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33220, 32731, 33022, 32470, 32486, 32834, 32577] avg=32768.0000 max=33220 max_rank=1 min=32470 min_rank=4 imbalance_ratio=1.013794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33220, 32731, 33022, 32470, 32486, 32834, 32577] avg=32768.0000 max=33220 max_rank=1 min=32470 min_rank=4 imbalance_ratio=1.013794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32505, 32735, 33187, 32674] avg=32768.0000 max=33187 max_rank=6 min=32505 min_rank=4 imbalance_ratio=1.012787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=28 tokens_per_ep_rank=[32534, 32853, 33234, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33234 max_rank=2 min=32534 min_rank=0 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33220, 32731, 33022, 32470, 32486, 32834, 32577] avg=32768.0000 max=33220 max_rank=1 min=32470 min_rank=4 imbalance_ratio=1.013794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] avg=32768.0000 max=33096 max_rank=1 min=32486 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124 max_load_reduction_pct=0.003733 moved_to_replicas=124 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] avg=32768.0000 max=33096 max_rank=1 min=32486 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124 max_load_reduction_pct=0.003733 moved_to_replicas=124 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] avg=32768.0000 max=33096 max_rank=6 min=32596 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=91 max_load_reduction_pct=0.002742 moved_to_replicas=91 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32505, 32735, 33187, 32674] avg=32768.0000 max=33187 max_rank=6 min=32505 min_rank=4 imbalance_ratio=1.012787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=28 tokens_per_ep_rank=[32672, 32853, 33096, 32660, 32691, 32764, 32626, 32782] avg=32768.0000 max=33096 max_rank=2 min=32626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] avg=32768.0000 max=33096 max_rank=1 min=32486 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124 max_load_reduction_pct=0.003733 moved_to_replicas=124 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33220, 32731, 33022, 32470, 32486, 32834, 32577] avg=32768.0000 max=33220 max_rank=1 min=32470 min_rank=4 imbalance_ratio=1.013794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] avg=32768.0000 max=33096 max_rank=6 min=32596 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=91 max_load_reduction_pct=0.002742 moved_to_replicas=91 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] avg=32768.0000 max=33096 max_rank=1 min=32486 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124 max_load_reduction_pct=0.003733 moved_to_replicas=124 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=29 tokens_per_ep_rank=[32984, 32822, 32679, 32538, 32625, 32849, 32826, 32821] avg=32768.0000 max=32984 max_rank=0 min=32538 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33220, 32731, 33022, 32470, 32486, 32834, 32577] avg=32768.0000 max=33220 max_rank=1 min=32470 min_rank=4 imbalance_ratio=1.013794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32505, 32735, 33187, 32674] avg=32768.0000 max=33187 max_rank=6 min=32505 min_rank=4 imbalance_ratio=1.012787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] avg=32768.0000 max=33096 max_rank=1 min=32486 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124 max_load_reduction_pct=0.003733 moved_to_replicas=124 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33220, 32731, 33022, 32470, 32486, 32834, 32577] avg=32768.0000 max=33220 max_rank=1 min=32470 min_rank=4 imbalance_ratio=1.013794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=30 tokens_per_ep_rank=[32837, 32867, 32690, 32649, 32596, 32735, 33096, 32674] avg=32768.0000 max=33096 max_rank=6 min=32596 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=91 max_load_reduction_pct=0.002742 moved_to_replicas=91 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32473, 33058, 33172, 32754] avg=32768.0000 max=33172 max_rank=6 min=32473 min_rank=4 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] avg=32768.0000 max=33096 max_rank=1 min=32486 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124 max_load_reduction_pct=0.003733 moved_to_replicas=124 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] avg=32768.0000 max=33096 max_rank=6 min=32482 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=31 tokens_per_ep_rank=[33002, 32715, 32921, 32656, 32877, 32794, 32403, 32776] avg=32768.0000 max=33002 max_rank=0 min=32403 min_rank=6 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32473, 33058, 33172, 32754] avg=32768.0000 max=33172 max_rank=6 min=32473 min_rank=4 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33176, 32691, 32756, 32586, 32658, 32567, 32759] avg=32768.0000 max=33176 max_rank=1 min=32567 min_rank=6 imbalance_ratio=1.012451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32473, 33058, 33172, 32754] avg=32768.0000 max=33172 max_rank=6 min=32473 min_rank=4 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32473, 33058, 33172, 32754] avg=32768.0000 max=33172 max_rank=6 min=32473 min_rank=4 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] avg=32768.0000 max=33096 max_rank=6 min=32482 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] avg=32768.0000 max=33096 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=80 max_load_reduction_pct=0.002411 moved_to_replicas=80 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33220, 32731, 33022, 32470, 32486, 32834, 32577] avg=32768.0000 max=33220 max_rank=1 min=32470 min_rank=4 imbalance_ratio=1.013794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] avg=32768.0000 max=33096 max_rank=6 min=32482 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] avg=32768.0000 max=33096 max_rank=6 min=32482 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=32 tokens_per_ep_rank=[32804, 33096, 32731, 33022, 32594, 32486, 32834, 32577] avg=32768.0000 max=33096 max_rank=1 min=32486 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=124 max_load_reduction_pct=0.003733 moved_to_replicas=124 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33176, 32691, 32756, 32586, 32658, 32567, 32759] avg=32768.0000 max=33176 max_rank=1 min=32567 min_rank=6 imbalance_ratio=1.012451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32473, 33058, 33172, 32754] avg=32768.0000 max=33172 max_rank=6 min=32473 min_rank=4 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33176, 32691, 32756, 32586, 32658, 32567, 32759] avg=32768.0000 max=33176 max_rank=1 min=32567 min_rank=6 imbalance_ratio=1.012451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33176, 32691, 32756, 32586, 32658, 32567, 32759] avg=32768.0000 max=33176 max_rank=1 min=32567 min_rank=6 imbalance_ratio=1.012451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] avg=32768.0000 max=33096 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=80 max_load_reduction_pct=0.002411 moved_to_replicas=80 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] avg=32768.0000 max=33096 max_rank=6 min=32482 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] avg=32768.0000 max=33096 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=80 max_load_reduction_pct=0.002411 moved_to_replicas=80 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] avg=32768.0000 max=33096 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=80 max_load_reduction_pct=0.002411 moved_to_replicas=80 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=33 tokens_per_ep_rank=[32574, 32761, 32794, 32760, 32974, 32771, 32894, 32616] avg=32768.0000 max=32974 max_rank=4 min=32574 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33176, 32691, 32756, 32586, 32658, 32567, 32759] avg=32768.0000 max=33176 max_rank=1 min=32567 min_rank=6 imbalance_ratio=1.012451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32473, 33058, 33172, 32754] avg=32768.0000 max=33172 max_rank=6 min=32473 min_rank=4 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] avg=32768.0000 max=33096 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=80 max_load_reduction_pct=0.002411 moved_to_replicas=80 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] avg=32768.0000 max=33096 max_rank=6 min=32482 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=34 tokens_per_ep_rank=[32976, 32659, 32692, 32376, 32885, 32870, 32992, 32694] avg=32768.0000 max=32992 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32473, 33058, 33172, 32754] avg=32768.0000 max=33172 max_rank=6 min=32473 min_rank=4 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33176, 32691, 32756, 32586, 32658, 32567, 32759] avg=32768.0000 max=33176 max_rank=1 min=32567 min_rank=6 imbalance_ratio=1.012451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] avg=32768.0000 max=33096 max_rank=6 min=32482 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] avg=32768.0000 max=33096 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=80 max_load_reduction_pct=0.002411 moved_to_replicas=80 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=35 tokens_per_ep_rank=[32611, 32870, 32732, 32743, 32944, 32526, 32889, 32829] avg=32768.0000 max=32944 max_rank=4 min=32526 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33176, 32691, 32756, 32586, 32658, 32567, 32759] avg=32768.0000 max=33176 max_rank=1 min=32567 min_rank=6 imbalance_ratio=1.012451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] avg=32768.0000 max=33096 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=80 max_load_reduction_pct=0.002411 moved_to_replicas=80 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32473, 33058, 33172, 32754] avg=32768.0000 max=33172 max_rank=6 min=32473 min_rank=4 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=36 tokens_per_ep_rank=[32482, 32638, 32603, 32964, 32549, 33058, 33096, 32754] avg=32768.0000 max=33096 max_rank=6 min=32482 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32384, 33155, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33155 max_rank=2 min=32384 min_rank=1 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33176, 32691, 32756, 32586, 32658, 32567, 32759] avg=32768.0000 max=33176 max_rank=1 min=32567 min_rank=6 imbalance_ratio=1.012451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=37 tokens_per_ep_rank=[32951, 33096, 32691, 32756, 32586, 32658, 32647, 32759] avg=32768.0000 max=33096 max_rank=1 min=32586 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=80 max_load_reduction_pct=0.002411 moved_to_replicas=80 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32520, 32598, 32717, 32733, 33307, 32631, 32827] avg=32768.0000 max=33307 max_rank=5 min=32520 min_rank=1 imbalance_ratio=1.016449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32384, 33155, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33155 max_rank=2 min=32384 min_rank=1 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32384, 33155, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33155 max_rank=2 min=32384 min_rank=1 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32384, 33155, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33155 max_rank=2 min=32384 min_rank=1 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] avg=32768.0000 max=33096 max_rank=5 min=32598 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=211 max_load_reduction_pct=0.006335 moved_to_replicas=211 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=38 tokens_per_ep_rank=[32769, 32633, 32705, 32994, 32614, 32862, 32823, 32744] avg=32768.0000 max=32994 max_rank=3 min=32614 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32520, 32598, 32717, 32733, 33307, 32631, 32827] avg=32768.0000 max=33307 max_rank=5 min=32520 min_rank=1 imbalance_ratio=1.016449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32520, 32598, 32717, 32733, 33307, 32631, 32827] avg=32768.0000 max=33307 max_rank=5 min=32520 min_rank=1 imbalance_ratio=1.016449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32384, 33155, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33155 max_rank=2 min=32384 min_rank=1 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32520, 32598, 32717, 32733, 33307, 32631, 32827] avg=32768.0000 max=33307 max_rank=5 min=32520 min_rank=1 imbalance_ratio=1.016449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] avg=32768.0000 max=33096 max_rank=5 min=32598 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=211 max_load_reduction_pct=0.006335 moved_to_replicas=211 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] avg=32768.0000 max=33096 max_rank=5 min=32598 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=211 max_load_reduction_pct=0.006335 moved_to_replicas=211 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] avg=32768.0000 max=33096 max_rank=5 min=32598 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=211 max_load_reduction_pct=0.006335 moved_to_replicas=211 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=39 tokens_per_ep_rank=[33028, 32937, 32529, 32606, 32850, 32760, 32889, 32545] avg=32768.0000 max=33028 max_rank=0 min=32529 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32520, 32598, 32717, 32733, 33307, 32631, 32827] avg=32768.0000 max=33307 max_rank=5 min=32520 min_rank=1 imbalance_ratio=1.016449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32384, 33155, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33155 max_rank=2 min=32384 min_rank=1 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] avg=32768.0000 max=33096 max_rank=5 min=32598 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=211 max_load_reduction_pct=0.006335 moved_to_replicas=211 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=40 tokens_per_ep_rank=[32608, 32793, 32587, 32895, 32844, 32815, 32838, 32764] avg=32768.0000 max=32895 max_rank=3 min=32587 min_rank=2 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32384, 33155, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33155 max_rank=2 min=32384 min_rank=1 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32520, 32598, 32717, 32733, 33307, 32631, 32827] avg=32768.0000 max=33307 max_rank=5 min=32520 min_rank=1 imbalance_ratio=1.016449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] avg=32768.0000 max=33096 max_rank=5 min=32598 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=211 max_load_reduction_pct=0.006335 moved_to_replicas=211 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32520, 32598, 32717, 32733, 33307, 32631, 32827] avg=32768.0000 max=33307 max_rank=5 min=32520 min_rank=1 imbalance_ratio=1.016449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=41 tokens_per_ep_rank=[32795, 32490, 32625, 33090, 32621, 32976, 32920, 32627] avg=32768.0000 max=33090 max_rank=3 min=32490 min_rank=1 imbalance_ratio=1.009827 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] avg=32768.0000 max=33096 max_rank=5 min=32598 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=211 max_load_reduction_pct=0.006335 moved_to_replicas=211 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32384, 33155, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33155 max_rank=2 min=32384 min_rank=1 imbalance_ratio=1.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32471, 32814, 33107, 32684, 33011] avg=32768.0000 max=33107 max_rank=5 min=32471 min_rank=3 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=42 tokens_per_ep_rank=[32667, 32443, 33096, 32578, 32907, 32912, 32719, 32822] avg=32768.0000 max=33096 max_rank=2 min=32443 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=59 max_load_reduction_pct=0.001780 moved_to_replicas=59 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] avg=32768.0000 max=33096 max_rank=5 min=32482 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32520, 32598, 32717, 32733, 33307, 32631, 32827] avg=32768.0000 max=33307 max_rank=5 min=32520 min_rank=1 imbalance_ratio=1.016449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32471, 32814, 33107, 32684, 33011] avg=32768.0000 max=33107 max_rank=5 min=32471 min_rank=3 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32471, 32814, 33107, 32684, 33011] avg=32768.0000 max=33107 max_rank=5 min=32471 min_rank=3 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=43 tokens_per_ep_rank=[32811, 32731, 32598, 32717, 32733, 33096, 32631, 32827] avg=32768.0000 max=33096 max_rank=5 min=32598 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=211 max_load_reduction_pct=0.006335 moved_to_replicas=211 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] avg=32768.0000 max=33096 max_rank=5 min=32482 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32471, 32814, 33107, 32684, 33011] avg=32768.0000 max=33107 max_rank=5 min=32471 min_rank=3 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] avg=32768.0000 max=33096 max_rank=5 min=32482 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] avg=32768.0000 max=33096 max_rank=5 min=32482 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=44 tokens_per_ep_rank=[32824, 32811, 32568, 32732, 32609, 32779, 32783, 33038] avg=32768.0000 max=33038 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32471, 32814, 33107, 32684, 33011] avg=32768.0000 max=33107 max_rank=5 min=32471 min_rank=3 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] avg=32768.0000 max=33096 max_rank=5 min=32482 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=45 tokens_per_ep_rank=[32675, 32858, 32932, 32478, 32830, 32775, 32698, 32898] avg=32768.0000 max=32932 max_rank=2 min=32478 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32471, 32814, 33107, 32684, 33011] avg=32768.0000 max=33107 max_rank=5 min=32471 min_rank=3 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] avg=32768.0000 max=33096 max_rank=5 min=32482 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32471, 32814, 33107, 32684, 33011] avg=32768.0000 max=33107 max_rank=5 min=32471 min_rank=3 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=46 tokens_per_ep_rank=[32599, 33063, 33003, 33019, 32427, 32647, 32632, 32754] avg=32768.0000 max=33063 max_rank=1 min=32427 min_rank=4 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] avg=32768.0000 max=33096 max_rank=5 min=32482 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=47 tokens_per_ep_rank=[32849, 32650, 32639, 32700, 32681, 33031, 32730, 32864] avg=32768.0000 max=33031 max_rank=5 min=32639 min_rank=2 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32471, 32814, 33107, 32684, 33011] avg=32768.0000 max=33107 max_rank=5 min=32471 min_rank=3 imbalance_ratio=1.010345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=48 tokens_per_ep_rank=[32633, 32703, 32721, 32482, 32814, 33096, 32684, 33011] avg=32768.0000 max=33096 max_rank=5 min=32482 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11 max_load_reduction_pct=0.000332 moved_to_replicas=11 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=48 total_expected_recv=1574357 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=340352 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=48 total_expected_recv=1573610 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=48 total_expected_recv=1574562 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341632 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=48 total_expected_recv=1570705 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=347008 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=48 total_expected_recv=1571682 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33005 required_capacity=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=48 total_expected_recv=1572572 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344704 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=48 total_expected_recv=1573789 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33046 required_capacity=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=48 total_expected_recv=1571635 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346496 peak_demand=33090 required_capacity=1.009827 + [2026-08-26 15:48:18] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 705.6 | throughput per GPU (TFLOP/s/GPU): 126.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.293148E+00 | loss scale: 1.0 | grad norm: 6.795 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32941 expected_overflow=0 exact_recv_per_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32941 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32959 expected_overflow=0 exact_recv_per_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32550 expected_overflow=0 exact_recv_per_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32959 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32550 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32559 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32938 expected_overflow=0 exact_recv_per_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32938 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33084 expected_overflow=0 exact_recv_per_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33084 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32516 expected_overflow=0 exact_recv_per_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33009 expected_overflow=0 exact_recv_per_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32516 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33009 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32975 expected_overflow=0 exact_recv_per_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32975 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32991 expected_overflow=0 exact_recv_per_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32991 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33038 expected_overflow=0 exact_recv_per_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=40960 expected_recv=33038 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32524 expected_overflow=0 exact_recv_per_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32524 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32570 expected_overflow=0 exact_recv_per_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32570 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32629 expected_overflow=0 exact_recv_per_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32629 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33041 expected_overflow=0 exact_recv_per_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=40960 expected_recv=33041 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33039 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32341 expected_overflow=0 exact_recv_per_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32341 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32514 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32548 expected_overflow=0 exact_recv_per_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32548 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33091 expected_overflow=0 exact_recv_per_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33091 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33066 expected_overflow=0 exact_recv_per_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=40960 expected_recv=33066 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33014 expected_overflow=0 exact_recv_per_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32286 expected_overflow=0 exact_recv_per_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32378 expected_overflow=0 exact_recv_per_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33046 expected_overflow=0 exact_recv_per_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33014 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=40960 expected_recv=32286 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32378 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33046 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32915 expected_overflow=0 exact_recv_per_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32915 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32397 expected_overflow=0 exact_recv_per_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=23 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32397 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33091 expected_overflow=0 exact_recv_per_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=23 calls=1 budget=40960 expected_recv=33091 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=25 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=24 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=25 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=26 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=29 calls=1 budget=40960 expected_recv=33026 expected_overflow=0 exact_recv_per_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=29 calls=1 budget=40960 expected_recv=33026 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=27 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=28 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=30 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32515 expected_overflow=0 exact_recv_per_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=29 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32515 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=30 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33085 expected_overflow=0 exact_recv_per_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=32 calls=1 budget=40960 expected_recv=33085 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=31 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32514 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33091 expected_overflow=0 exact_recv_per_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=32 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=34 calls=1 budget=40960 expected_recv=33091 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33086 expected_overflow=0 exact_recv_per_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=33 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=35 calls=1 budget=40960 expected_recv=33086 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=34 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32323 expected_overflow=0 exact_recv_per_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32323 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=35 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=36 calls=1 budget=40960 expected_recv=32553 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=37 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=38 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33004 expected_overflow=0 exact_recv_per_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=40 calls=1 budget=40960 expected_recv=33004 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32429 expected_overflow=0 exact_recv_per_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=39 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32429 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=39 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32553 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33021 expected_overflow=0 exact_recv_per_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=41 calls=1 budget=40960 expected_recv=33021 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=40 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33092 expected_overflow=0 exact_recv_per_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=42 calls=1 budget=40960 expected_recv=33092 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=41 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=44 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32493 expected_overflow=0 exact_recv_per_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=42 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32493 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32973 expected_overflow=0 exact_recv_per_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32973 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=43 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=43 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32452 expected_overflow=0 exact_recv_per_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32452 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=44 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33055 expected_overflow=0 exact_recv_per_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=46 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=47 calls=1 budget=40960 expected_recv=33055 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=45 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33061 expected_overflow=0 exact_recv_per_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33062 expected_overflow=0 exact_recv_per_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33061 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=46 calls=1 budget=40960 expected_recv=33062 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=47 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32596, 32665, 32941, 32773, 32764, 32795, 32670, 32940] avg=32768.0000 max=32941 max_rank=2 min=32596 min_rank=0 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32783, 32790, 32673, 32933, 32959, 32550, 32657, 32799] avg=32768.0000 max=32959 max_rank=4 min=32550 min_rank=5 imbalance_ratio=1.005829 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32781, 32962, 32735, 32793, 32678, 32805, 32686, 32704] avg=32768.0000 max=32962 max_rank=1 min=32678 min_rank=4 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32790, 32559, 32812, 32794, 32642, 32785, 32824, 32938] avg=32768.0000 max=32938 max_rank=7 min=32559 min_rank=1 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32644, 33084, 32523, 32830, 33009, 32835, 32703, 32516] avg=32768.0000 max=33084 max_rank=1 min=32516 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32485, 32486, 32826, 32796, 32703, 32642, 33221] avg=32768.0000 max=33221 max_rank=7 min=32485 min_rank=1 imbalance_ratio=1.013824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32485, 32486, 32826, 32796, 32703, 32642, 33221] avg=32768.0000 max=33221 max_rank=7 min=32485 min_rank=1 imbalance_ratio=1.013824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32544, 32586, 32802, 32781, 32692, 32975, 32899, 32865] avg=32768.0000 max=32975 max_rank=5 min=32544 min_rank=0 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] avg=32768.0000 max=33096 max_rank=7 min=32486 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=125 max_load_reduction_pct=0.003763 moved_to_replicas=125 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32485, 32486, 32826, 32796, 32703, 32642, 33221] avg=32768.0000 max=33221 max_rank=7 min=32485 min_rank=1 imbalance_ratio=1.013824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32485, 32486, 32826, 32796, 32703, 32642, 33221] avg=32768.0000 max=33221 max_rank=7 min=32485 min_rank=1 imbalance_ratio=1.013824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] avg=32768.0000 max=33096 max_rank=7 min=32486 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=125 max_load_reduction_pct=0.003763 moved_to_replicas=125 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32485, 32486, 32826, 32796, 32703, 32642, 33221] avg=32768.0000 max=33221 max_rank=7 min=32485 min_rank=1 imbalance_ratio=1.013824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] avg=32768.0000 max=33096 max_rank=7 min=32486 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=125 max_load_reduction_pct=0.003763 moved_to_replicas=125 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] avg=32768.0000 max=33096 max_rank=7 min=32486 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=125 max_load_reduction_pct=0.003763 moved_to_replicas=125 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32485, 32486, 32826, 32796, 32703, 32642, 33221] avg=32768.0000 max=33221 max_rank=7 min=32485 min_rank=1 imbalance_ratio=1.013824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] avg=32768.0000 max=33096 max_rank=7 min=32486 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=125 max_load_reduction_pct=0.003763 moved_to_replicas=125 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32778, 32864, 32721, 32788, 32906, 32799, 32848, 32440] avg=32768.0000 max=32906 max_rank=4 min=32440 min_rank=7 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] avg=32768.0000 max=33096 max_rank=7 min=32486 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=125 max_load_reduction_pct=0.003763 moved_to_replicas=125 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32891, 32772, 32800, 32824, 32695, 32563, 32873, 32726] avg=32768.0000 max=32891 max_rank=0 min=32563 min_rank=5 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32485, 32486, 32826, 32796, 32703, 32642, 33221] avg=32768.0000 max=33221 max_rank=7 min=32485 min_rank=1 imbalance_ratio=1.013824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32485, 32486, 32826, 32796, 32703, 32642, 33221] avg=32768.0000 max=33221 max_rank=7 min=32485 min_rank=1 imbalance_ratio=1.013824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] avg=32768.0000 max=33096 max_rank=7 min=32486 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=125 max_load_reduction_pct=0.003763 moved_to_replicas=125 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[32985, 32610, 32486, 32826, 32796, 32703, 32642, 33096] avg=32768.0000 max=33096 max_rank=7 min=32486 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=125 max_load_reduction_pct=0.003763 moved_to_replicas=125 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32854, 32958, 32690, 32985, 32606, 32596, 32991, 32464] avg=32768.0000 max=32991 max_rank=6 min=32464 min_rank=7 imbalance_ratio=1.006805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32661, 32813, 33038, 32760, 32831, 32524, 32615, 32902] avg=32768.0000 max=33038 max_rank=2 min=32524 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[32710, 32967, 32736, 32921, 32703, 32956, 32660, 32491] avg=32768.0000 max=32967 max_rank=1 min=32491 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32772, 32620, 32526, 32828, 32966, 32925, 32760, 32747] avg=32768.0000 max=32966 max_rank=4 min=32526 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32562, 32604, 32683, 32798, 33099] avg=32768.0000 max=33099 max_rank=7 min=32562 min_rank=3 imbalance_ratio=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32562, 32604, 32683, 32798, 33099] avg=32768.0000 max=33099 max_rank=7 min=32562 min_rank=3 imbalance_ratio=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32562, 32604, 32683, 32798, 33099] avg=32768.0000 max=33099 max_rank=7 min=32562 min_rank=3 imbalance_ratio=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] avg=32768.0000 max=33096 max_rank=7 min=32565 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=3 max_load_reduction_pct=0.000091 moved_to_replicas=3 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] avg=32768.0000 max=33096 max_rank=7 min=32565 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=3 max_load_reduction_pct=0.000091 moved_to_replicas=3 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32562, 32604, 32683, 32798, 33099] avg=32768.0000 max=33099 max_rank=7 min=32562 min_rank=3 imbalance_ratio=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32562, 32604, 32683, 32798, 33099] avg=32768.0000 max=33099 max_rank=7 min=32562 min_rank=3 imbalance_ratio=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] avg=32768.0000 max=33096 max_rank=7 min=32565 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=3 max_load_reduction_pct=0.000091 moved_to_replicas=3 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32882, 32822, 32965, 33034, 32761, 32579, 32664, 32437] avg=32768.0000 max=33034 max_rank=3 min=32437 min_rank=7 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] avg=32768.0000 max=33096 max_rank=7 min=32565 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=3 max_load_reduction_pct=0.000091 moved_to_replicas=3 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] avg=32768.0000 max=33096 max_rank=7 min=32565 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=3 max_load_reduction_pct=0.000091 moved_to_replicas=3 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32562, 32604, 32683, 32798, 33099] avg=32768.0000 max=33099 max_rank=7 min=32562 min_rank=3 imbalance_ratio=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] avg=32768.0000 max=33096 max_rank=7 min=32565 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=3 max_load_reduction_pct=0.000091 moved_to_replicas=3 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32545, 32570, 32892, 32879, 32612, 32981, 32816, 32849] avg=32768.0000 max=32981 max_rank=5 min=32545 min_rank=0 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32687, 32720, 32604, 33041, 32879, 32864, 32720, 32629] avg=32768.0000 max=33041 max_rank=3 min=32604 min_rank=2 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32562, 32604, 32683, 32798, 33099] avg=32768.0000 max=33099 max_rank=7 min=32562 min_rank=3 imbalance_ratio=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32562, 32604, 32683, 32798, 33099] avg=32768.0000 max=33099 max_rank=7 min=32562 min_rank=3 imbalance_ratio=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] avg=32768.0000 max=33096 max_rank=7 min=32565 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=3 max_load_reduction_pct=0.000091 moved_to_replicas=3 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[33040, 32751, 32607, 32565, 32604, 32683, 32798, 33096] avg=32768.0000 max=33096 max_rank=7 min=32565 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=3 max_load_reduction_pct=0.000091 moved_to_replicas=3 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32580, 32645, 33118] avg=32768.0000 max=33118 max_rank=7 min=32580 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[33023, 32681, 32671, 32825, 32633, 32931, 32341, 33039] avg=32768.0000 max=33039 max_rank=7 min=32341 min_rank=6 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32580, 32645, 33118] avg=32768.0000 max=33118 max_rank=7 min=32580 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] avg=32768.0000 max=33096 max_rank=7 min=32586 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32580, 32645, 33118] avg=32768.0000 max=33118 max_rank=7 min=32580 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] avg=32768.0000 max=33096 max_rank=7 min=32586 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] avg=32768.0000 max=33096 max_rank=7 min=32586 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32580, 32645, 33118] avg=32768.0000 max=33118 max_rank=7 min=32580 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32580, 32645, 33118] avg=32768.0000 max=33118 max_rank=7 min=32580 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[32761, 32690, 32830, 32748, 32932, 32514, 32788, 32881] avg=32768.0000 max=32932 max_rank=4 min=32514 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] avg=32768.0000 max=33096 max_rank=7 min=32586 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] avg=32768.0000 max=33096 max_rank=7 min=32586 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32580, 32645, 33118] avg=32768.0000 max=33118 max_rank=7 min=32580 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[32812, 32637, 32918, 32686, 32757, 32886, 32755, 32693] avg=32768.0000 max=32918 max_rank=2 min=32637 min_rank=1 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] avg=32768.0000 max=33096 max_rank=7 min=32586 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[32548, 32616, 32805, 32929, 32827, 32843, 32795, 32781] avg=32768.0000 max=32929 max_rank=3 min=32548 min_rank=0 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32580, 32645, 33118] avg=32768.0000 max=33118 max_rank=7 min=32580 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32580, 32645, 33118] avg=32768.0000 max=33118 max_rank=7 min=32580 min_rank=5 imbalance_ratio=1.010681 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] avg=32768.0000 max=33096 max_rank=7 min=32586 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[32643, 32586, 32984, 32769, 32819, 32602, 32645, 33096] avg=32768.0000 max=33096 max_rank=7 min=32586 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=22 max_load_reduction_pct=0.000664 moved_to_replicas=22 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=27 tokens_per_ep_rank=[33236, 32667, 32540, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33236 max_rank=0 min=32540 min_rank=2 imbalance_ratio=1.014282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[32653, 33066, 33091, 32286, 32673, 32663, 33034, 32678] avg=32768.0000 max=33091 max_rank=2 min=32286 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=27 tokens_per_ep_rank=[33236, 32667, 32540, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33236 max_rank=0 min=32540 min_rank=2 imbalance_ratio=1.014282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=27 tokens_per_ep_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33096 max_rank=0 min=32544 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=140 max_load_reduction_pct=0.004212 moved_to_replicas=140 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=27 tokens_per_ep_rank=[33236, 32667, 32540, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33236 max_rank=0 min=32540 min_rank=2 imbalance_ratio=1.014282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=27 tokens_per_ep_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33096 max_rank=0 min=32544 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=140 max_load_reduction_pct=0.004212 moved_to_replicas=140 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=27 tokens_per_ep_rank=[33236, 32667, 32540, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33236 max_rank=0 min=32540 min_rank=2 imbalance_ratio=1.014282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=27 tokens_per_ep_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33096 max_rank=0 min=32544 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=140 max_load_reduction_pct=0.004212 moved_to_replicas=140 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=27 tokens_per_ep_rank=[33236, 32667, 32540, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33236 max_rank=0 min=32540 min_rank=2 imbalance_ratio=1.014282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=27 tokens_per_ep_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33096 max_rank=0 min=32544 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=140 max_load_reduction_pct=0.004212 moved_to_replicas=140 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[32771, 32378, 32660, 33091, 33046, 32708, 33014, 32476] avg=32768.0000 max=33091 max_rank=3 min=32378 min_rank=1 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=27 tokens_per_ep_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33096 max_rank=0 min=32544 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=140 max_load_reduction_pct=0.004212 moved_to_replicas=140 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=27 tokens_per_ep_rank=[33236, 32667, 32540, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33236 max_rank=0 min=32540 min_rank=2 imbalance_ratio=1.014282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=25 tokens_per_ep_rank=[32701, 32999, 32915, 32567, 32751, 32838, 32976, 32397] avg=32768.0000 max=32999 max_rank=1 min=32397 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=27 tokens_per_ep_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33096 max_rank=0 min=32544 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=140 max_load_reduction_pct=0.004212 moved_to_replicas=140 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=26 tokens_per_ep_rank=[32783, 33005, 32529, 32853, 32924, 32471, 32784, 32795] avg=32768.0000 max=33005 max_rank=1 min=32471 min_rank=5 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=27 tokens_per_ep_rank=[33236, 32667, 32540, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33236 max_rank=0 min=32540 min_rank=2 imbalance_ratio=1.014282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=27 tokens_per_ep_rank=[33236, 32667, 32540, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33236 max_rank=0 min=32540 min_rank=2 imbalance_ratio=1.014282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=27 tokens_per_ep_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33096 max_rank=0 min=32544 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=140 max_load_reduction_pct=0.004212 moved_to_replicas=140 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=27 tokens_per_ep_rank=[33096, 32667, 32680, 32544, 32693, 32850, 32744, 32870] avg=32768.0000 max=33096 max_rank=0 min=32544 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=140 max_load_reduction_pct=0.004212 moved_to_replicas=140 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=28 tokens_per_ep_rank=[32547, 32770, 32790, 32962, 32669, 32760, 32704, 32942] avg=32768.0000 max=32962 max_rank=3 min=32547 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=29 tokens_per_ep_rank=[32811, 32812, 32879, 32714, 32698, 32775, 32823, 32632] avg=32768.0000 max=32879 max_rank=2 min=32632 min_rank=7 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=30 tokens_per_ep_rank=[32773, 32657, 33026, 32871, 32500, 32794, 32817, 32706] avg=32768.0000 max=33026 max_rank=2 min=32500 min_rank=4 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=31 tokens_per_ep_rank=[32515, 32858, 32912, 32572, 32605, 33045, 32876, 32761] avg=32768.0000 max=33045 max_rank=5 min=32515 min_rank=0 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=32 tokens_per_ep_rank=[32641, 32731, 32879, 32982, 32573, 32837, 32834, 32667] avg=32768.0000 max=32982 max_rank=3 min=32573 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=33 tokens_per_ep_rank=[32966, 32671, 32472, 32824, 32658, 32634, 33085, 32834] avg=32768.0000 max=33085 max_rank=6 min=32472 min_rank=2 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33276, 32493] avg=32768.0000 max=33276 max_rank=6 min=32493 min_rank=7 imbalance_ratio=1.015503 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=34 tokens_per_ep_rank=[32823, 32767, 32612, 32823, 32813, 32514, 32899, 32893] avg=32768.0000 max=32899 max_rank=6 min=32514 min_rank=5 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] avg=32768.0000 max=33096 max_rank=6 min=32529 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180 max_load_reduction_pct=0.005409 moved_to_replicas=180 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33276, 32493] avg=32768.0000 max=33276 max_rank=6 min=32493 min_rank=7 imbalance_ratio=1.015503 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33276, 32493] avg=32768.0000 max=33276 max_rank=6 min=32493 min_rank=7 imbalance_ratio=1.015503 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33276, 32493] avg=32768.0000 max=33276 max_rank=6 min=32493 min_rank=7 imbalance_ratio=1.015503 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=40 tokens_per_ep_rank=[33210, 32958, 32643, 32696, 32759, 32692, 32670, 32516] avg=32768.0000 max=33210 max_rank=0 min=32516 min_rank=7 imbalance_ratio=1.013489 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] avg=32768.0000 max=33096 max_rank=6 min=32529 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180 max_load_reduction_pct=0.005409 moved_to_replicas=180 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] avg=32768.0000 max=33096 max_rank=6 min=32529 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180 max_load_reduction_pct=0.005409 moved_to_replicas=180 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=35 tokens_per_ep_rank=[32723, 32558, 32895, 32539, 32808, 32933, 32597, 33091] avg=32768.0000 max=33091 max_rank=7 min=32539 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] avg=32768.0000 max=33096 max_rank=6 min=32529 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180 max_load_reduction_pct=0.005409 moved_to_replicas=180 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=40 tokens_per_ep_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] avg=32768.0000 max=33096 max_rank=0 min=32630 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=114 max_load_reduction_pct=0.003433 moved_to_replicas=114 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33276, 32493] avg=32768.0000 max=33276 max_rank=6 min=32493 min_rank=7 imbalance_ratio=1.015503 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=40 tokens_per_ep_rank=[33210, 32958, 32643, 32696, 32759, 32692, 32670, 32516] avg=32768.0000 max=33210 max_rank=0 min=32516 min_rank=7 imbalance_ratio=1.013489 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=40 tokens_per_ep_rank=[33210, 32958, 32643, 32696, 32759, 32692, 32670, 32516] avg=32768.0000 max=33210 max_rank=0 min=32516 min_rank=7 imbalance_ratio=1.013489 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=40 tokens_per_ep_rank=[33210, 32958, 32643, 32696, 32759, 32692, 32670, 32516] avg=32768.0000 max=33210 max_rank=0 min=32516 min_rank=7 imbalance_ratio=1.013489 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] avg=32768.0000 max=33096 max_rank=6 min=32529 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180 max_load_reduction_pct=0.005409 moved_to_replicas=180 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=40 tokens_per_ep_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] avg=32768.0000 max=33096 max_rank=0 min=32630 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=114 max_load_reduction_pct=0.003433 moved_to_replicas=114 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=40 tokens_per_ep_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] avg=32768.0000 max=33096 max_rank=0 min=32630 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=114 max_load_reduction_pct=0.003433 moved_to_replicas=114 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=40 tokens_per_ep_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] avg=32768.0000 max=33096 max_rank=0 min=32630 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=114 max_load_reduction_pct=0.003433 moved_to_replicas=114 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=36 tokens_per_ep_rank=[32323, 32775, 32847, 32582, 32823, 32907, 32801, 33086] avg=32768.0000 max=33086 max_rank=7 min=32323 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33276, 32493] avg=32768.0000 max=33276 max_rank=6 min=32493 min_rank=7 imbalance_ratio=1.015503 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=40 tokens_per_ep_rank=[33210, 32958, 32643, 32696, 32759, 32692, 32670, 32516] avg=32768.0000 max=33210 max_rank=0 min=32516 min_rank=7 imbalance_ratio=1.013489 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] avg=32768.0000 max=33096 max_rank=6 min=32529 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180 max_load_reduction_pct=0.005409 moved_to_replicas=180 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=40 tokens_per_ep_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] avg=32768.0000 max=33096 max_rank=0 min=32630 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=114 max_load_reduction_pct=0.003433 moved_to_replicas=114 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=40 tokens_per_ep_rank=[33210, 32958, 32643, 32696, 32759, 32692, 32670, 32516] avg=32768.0000 max=33210 max_rank=0 min=32516 min_rank=7 imbalance_ratio=1.013489 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=37 tokens_per_ep_rank=[32726, 32908, 32802, 32553, 32587, 32876, 32860, 32832] avg=32768.0000 max=32908 max_rank=1 min=32553 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=40 tokens_per_ep_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] avg=32768.0000 max=33096 max_rank=0 min=32630 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=114 max_load_reduction_pct=0.003433 moved_to_replicas=114 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=38 tokens_per_ep_rank=[32988, 32814, 32738, 32466, 32744, 32712, 32952, 32730] avg=32768.0000 max=32988 max_rank=0 min=32466 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33276, 32493] avg=32768.0000 max=33276 max_rank=6 min=32493 min_rank=7 imbalance_ratio=1.015503 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33329, 32597, 32785, 32704, 32741] avg=32768.0000 max=33329 max_rank=3 min=32597 min_rank=4 imbalance_ratio=1.017120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] avg=32768.0000 max=33096 max_rank=6 min=32529 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180 max_load_reduction_pct=0.005409 moved_to_replicas=180 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33276, 32493] avg=32768.0000 max=33276 max_rank=6 min=32493 min_rank=7 imbalance_ratio=1.015503 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] avg=32768.0000 max=33096 max_rank=3 min=32616 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=233 max_load_reduction_pct=0.006991 moved_to_replicas=233 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=39 tokens_per_ep_rank=[32956, 32754, 32844, 32631, 32529, 32661, 33096, 32673] avg=32768.0000 max=33096 max_rank=6 min=32529 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=180 max_load_reduction_pct=0.005409 moved_to_replicas=180 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33329, 32597, 32785, 32704, 32741] avg=32768.0000 max=33329 max_rank=3 min=32597 min_rank=4 imbalance_ratio=1.017120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=40 tokens_per_ep_rank=[33210, 32958, 32643, 32696, 32759, 32692, 32670, 32516] avg=32768.0000 max=33210 max_rank=0 min=32516 min_rank=7 imbalance_ratio=1.013489 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33329, 32597, 32785, 32704, 32741] avg=32768.0000 max=33329 max_rank=3 min=32597 min_rank=4 imbalance_ratio=1.017120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33329, 32597, 32785, 32704, 32741] avg=32768.0000 max=33329 max_rank=3 min=32597 min_rank=4 imbalance_ratio=1.017120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] avg=32768.0000 max=33096 max_rank=3 min=32616 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=233 max_load_reduction_pct=0.006991 moved_to_replicas=233 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] avg=32768.0000 max=33096 max_rank=3 min=32616 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=233 max_load_reduction_pct=0.006991 moved_to_replicas=233 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=40 tokens_per_ep_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] avg=32768.0000 max=33096 max_rank=0 min=32630 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=114 max_load_reduction_pct=0.003433 moved_to_replicas=114 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=40 tokens_per_ep_rank=[33210, 32958, 32643, 32696, 32759, 32692, 32670, 32516] avg=32768.0000 max=33210 max_rank=0 min=32516 min_rank=7 imbalance_ratio=1.013489 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] avg=32768.0000 max=33096 max_rank=3 min=32616 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=233 max_load_reduction_pct=0.006991 moved_to_replicas=233 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33329, 32597, 32785, 32704, 32741] avg=32768.0000 max=33329 max_rank=3 min=32597 min_rank=4 imbalance_ratio=1.017120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=40 tokens_per_ep_rank=[33096, 32958, 32643, 32696, 32759, 32692, 32670, 32630] avg=32768.0000 max=33096 max_rank=0 min=32630 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=114 max_load_reduction_pct=0.003433 moved_to_replicas=114 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] avg=32768.0000 max=33096 max_rank=3 min=32616 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=233 max_load_reduction_pct=0.006991 moved_to_replicas=233 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33329, 32597, 32785, 32704, 32741] avg=32768.0000 max=33329 max_rank=3 min=32597 min_rank=4 imbalance_ratio=1.017120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=41 tokens_per_ep_rank=[32692, 32939, 32447, 32894, 32692, 33004, 32842, 32634] avg=32768.0000 max=33004 max_rank=5 min=32447 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] avg=32768.0000 max=33096 max_rank=3 min=32616 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=233 max_load_reduction_pct=0.006991 moved_to_replicas=233 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=42 tokens_per_ep_rank=[32878, 33021, 32875, 32581, 32819, 32553, 32429, 32988] avg=32768.0000 max=33021 max_rank=1 min=32429 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=43 tokens_per_ep_rank=[32628, 32894, 32589, 32663, 32877, 33092, 32607, 32794] avg=32768.0000 max=33092 max_rank=5 min=32589 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33329, 32597, 32785, 32704, 32741] avg=32768.0000 max=33329 max_rank=3 min=32597 min_rank=4 imbalance_ratio=1.017120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33329, 32597, 32785, 32704, 32741] avg=32768.0000 max=33329 max_rank=3 min=32597 min_rank=4 imbalance_ratio=1.017120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] avg=32768.0000 max=33096 max_rank=3 min=32616 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=233 max_load_reduction_pct=0.006991 moved_to_replicas=233 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=44 tokens_per_ep_rank=[32668, 32704, 32616, 33096, 32830, 32785, 32704, 32741] avg=32768.0000 max=33096 max_rank=3 min=32616 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=233 max_load_reduction_pct=0.006991 moved_to_replicas=233 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=45 tokens_per_ep_rank=[32600, 32973, 32779, 32845, 32857, 32493, 32562, 33035] avg=32768.0000 max=33035 max_rank=7 min=32493 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=46 tokens_per_ep_rank=[32790, 32634, 32980, 32836, 32871, 32661, 32672, 32700] avg=32768.0000 max=32980 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=47 tokens_per_ep_rank=[33062, 32569, 32921, 33061, 32590, 32452, 32691, 32798] avg=32768.0000 max=33062 max_rank=0 min=32452 min_rank=5 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=48 tokens_per_ep_rank=[32721, 32841, 32535, 32920, 32632, 32891, 32549, 33055] avg=32768.0000 max=33055 max_rank=7 min=32535 min_rank=2 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=48 total_expected_recv=1573567 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344704 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=48 total_expected_recv=1572715 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344320 peak_demand=33091 required_capacity=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=48 total_expected_recv=1572295 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346496 peak_demand=33092 required_capacity=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=48 total_expected_recv=1572772 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344704 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=48 total_expected_recv=1572092 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345472 peak_demand=33046 required_capacity=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=48 total_expected_recv=1573046 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343808 peak_demand=33084 required_capacity=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=48 total_expected_recv=1573764 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343168 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=48 total_expected_recv=1572661 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=347520 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:48:19] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 708.9 | throughput per GPU (TFLOP/s/GPU): 125.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.102908E+00 | loss scale: 1.0 | grad norm: 3.514 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32957 expected_overflow=0 exact_recv_per_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32957 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32503 expected_overflow=0 exact_recv_per_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32503 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33066 expected_overflow=0 exact_recv_per_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=33066 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33054 expected_overflow=0 exact_recv_per_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32578 expected_overflow=0 exact_recv_per_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=33054 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32578 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34688 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32990 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32495 expected_overflow=0 exact_recv_per_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32495 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33021 expected_overflow=0 exact_recv_per_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33021 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32600 expected_overflow=0 exact_recv_per_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32600 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33071 expected_overflow=0 exact_recv_per_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=40960 expected_recv=33071 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33037 expected_overflow=0 exact_recv_per_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32380 expected_overflow=0 exact_recv_per_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=40960 expected_recv=33037 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32380 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32363 expected_overflow=0 exact_recv_per_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32363 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32521 expected_overflow=0 exact_recv_per_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32521 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32451 expected_overflow=0 exact_recv_per_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32451 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32495 expected_overflow=0 exact_recv_per_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32495 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33021 expected_overflow=0 exact_recv_per_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33021 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33038 expected_overflow=0 exact_recv_per_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=40960 expected_recv=33038 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32973 expected_overflow=0 exact_recv_per_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32973 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=23 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=24 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=25 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33047 expected_overflow=0 exact_recv_per_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=27 calls=1 budget=40960 expected_recv=33047 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32493 expected_overflow=0 exact_recv_per_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=26 calls=1 budget=40960 expected_recv=32493 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32964 expected_overflow=0 exact_recv_per_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32926 expected_overflow=0 exact_recv_per_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32964 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32926 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=27 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32377 expected_overflow=0 exact_recv_per_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32964 expected_overflow=0 exact_recv_per_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32377 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32964 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=28 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33056 expected_overflow=0 exact_recv_per_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=28 calls=1 budget=40960 expected_recv=33056 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32602 expected_overflow=0 exact_recv_per_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32602 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=29 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=29 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32506 expected_overflow=0 exact_recv_per_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32506 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=30 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=31 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=32 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=33 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33067 expected_overflow=0 exact_recv_per_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=33 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=35 calls=1 budget=40960 expected_recv=33067 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32407 expected_overflow=0 exact_recv_per_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32407 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=34 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33032 expected_overflow=0 exact_recv_per_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=35 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33032 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=36 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=37 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33059 expected_overflow=0 exact_recv_per_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32497 expected_overflow=0 exact_recv_per_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=37 calls=1 budget=40960 expected_recv=33059 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32497 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32550 expected_overflow=0 exact_recv_per_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32550 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=38 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=40 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33079 expected_overflow=0 exact_recv_per_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=39 calls=1 budget=40960 expected_recv=33079 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=39 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=40 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33080 expected_overflow=0 exact_recv_per_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=41 calls=1 budget=40960 expected_recv=33080 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=41 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=42 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33008 expected_overflow=0 exact_recv_per_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=42 calls=1 budget=40960 expected_recv=33008 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33083 expected_overflow=0 exact_recv_per_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=45 calls=1 budget=40960 expected_recv=33083 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=43 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=44 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32949 expected_overflow=0 exact_recv_per_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32949 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32948 expected_overflow=0 exact_recv_per_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=45 calls=1 budget=40960 expected_recv=32948 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32555, 32568, 32750, 32957, 32714, 32832, 33113] avg=32768.0000 max=33113 max_rank=7 min=32555 min_rank=1 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32555, 32568, 32750, 32957, 32714, 32832, 33113] avg=32768.0000 max=33113 max_rank=7 min=32555 min_rank=1 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32555, 32568, 32750, 32957, 32714, 32832, 33113] avg=32768.0000 max=33113 max_rank=7 min=32555 min_rank=1 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32555, 32568, 32750, 32957, 32714, 32832, 33113] avg=32768.0000 max=33113 max_rank=7 min=32555 min_rank=1 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=46 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32555, 32568, 32750, 32957, 32714, 32832, 33113] avg=32768.0000 max=33113 max_rank=7 min=32555 min_rank=1 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] avg=32768.0000 max=33096 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] avg=32768.0000 max=33096 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] avg=32768.0000 max=33096 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] avg=32768.0000 max=33096 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] avg=32768.0000 max=33096 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=47 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32555, 32568, 32750, 32957, 32714, 32832, 33113] avg=32768.0000 max=33113 max_rank=7 min=32555 min_rank=1 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32555, 32568, 32750, 32957, 32714, 32832, 33113] avg=32768.0000 max=33113 max_rank=7 min=32555 min_rank=1 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] avg=32768.0000 max=33096 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] avg=32768.0000 max=33096 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32555, 32568, 32750, 32957, 32714, 32832, 33113] avg=32768.0000 max=33113 max_rank=7 min=32555 min_rank=1 imbalance_ratio=1.010529 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32655, 32572, 32568, 32750, 32957, 32714, 32832, 33096] avg=32768.0000 max=33096 max_rank=7 min=32568 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17 max_load_reduction_pct=0.000513 moved_to_replicas=17 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32636, 32715, 32715, 32743, 32805, 32908, 32894, 32728] avg=32768.0000 max=32908 max_rank=5 min=32636 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32861, 32734, 32832, 32693, 33066, 32579, 32503, 32876] avg=32768.0000 max=33066 max_rank=4 min=32503 min_rank=6 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32455, 33132, 33054] avg=32768.0000 max=33132 max_rank=6 min=32455 min_rank=5 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32455, 33132, 33054] avg=32768.0000 max=33132 max_rank=6 min=32455 min_rank=5 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32455, 33132, 33054] avg=32768.0000 max=33132 max_rank=6 min=32455 min_rank=5 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32455, 33132, 33054] avg=32768.0000 max=33132 max_rank=6 min=32455 min_rank=5 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] avg=32768.0000 max=33096 max_rank=6 min=32491 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32455, 33132, 33054] avg=32768.0000 max=33132 max_rank=6 min=32455 min_rank=5 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] avg=32768.0000 max=33096 max_rank=6 min=32491 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32654, 32815, 32745, 32868, 32784, 33072, 32491, 32715] avg=32768.0000 max=33072 max_rank=5 min=32491 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] avg=32768.0000 max=33096 max_rank=6 min=32491 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] avg=32768.0000 max=33096 max_rank=6 min=32491 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] avg=32768.0000 max=33096 max_rank=6 min=32491 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32691, 32526, 32761, 32934, 32927, 32704, 32861, 32740] avg=32768.0000 max=32934 max_rank=3 min=32526 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32455, 33132, 33054] avg=32768.0000 max=33132 max_rank=6 min=32455 min_rank=5 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32455, 33132, 33054] avg=32768.0000 max=33132 max_rank=6 min=32455 min_rank=5 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] avg=32768.0000 max=33096 max_rank=6 min=32491 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] avg=32768.0000 max=33096 max_rank=6 min=32491 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32455, 33132, 33054] avg=32768.0000 max=33132 max_rank=6 min=32455 min_rank=5 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32815, 32823, 32578, 32508, 32779, 32491, 33096, 33054] avg=32768.0000 max=33096 max_rank=6 min=32491 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32531, 32697, 32990, 32495, 32814, 32758, 32963, 32896] avg=32768.0000 max=32990 max_rank=2 min=32495 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[33021, 32432, 32563, 32785, 32863, 32741, 33045, 32694] avg=32768.0000 max=33045 max_rank=6 min=32432 min_rank=1 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32747, 32773, 32690, 32859, 32811, 32716, 32850, 32698] avg=32768.0000 max=32859 max_rank=3 min=32690 min_rank=2 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32672, 32513, 32823, 32822, 32842, 32944, 32733, 32795] avg=32768.0000 max=32944 max_rank=5 min=32513 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[32805, 32600, 33024, 32738, 32697, 32594, 32892, 32794] avg=32768.0000 max=33024 max_rank=2 min=32594 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[32746, 32695, 32865, 32908, 32937, 32744, 32605, 32644] avg=32768.0000 max=32937 max_rank=4 min=32605 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32751, 32679, 32594, 32563, 32937, 32716, 33071, 32833] avg=32768.0000 max=33071 max_rank=6 min=32563 min_rank=3 imbalance_ratio=1.009247 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[33126, 32683, 32678, 32597, 32624, 33115, 32659, 32662] avg=32768.0000 max=33126 max_rank=0 min=32597 min_rank=3 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[33126, 32683, 32678, 32597, 32624, 33115, 32659, 32662] avg=32768.0000 max=33126 max_rank=0 min=32597 min_rank=3 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[33126, 32683, 32678, 32597, 32624, 33115, 32659, 32662] avg=32768.0000 max=33126 max_rank=0 min=32597 min_rank=3 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[33126, 32683, 32678, 32597, 32624, 33115, 32659, 32662] avg=32768.0000 max=33126 max_rank=0 min=32597 min_rank=3 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[33126, 32683, 32678, 32597, 32624, 33115, 32659, 32662] avg=32768.0000 max=33126 max_rank=0 min=32597 min_rank=3 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32380, 33037, 32685, 32748, 32729, 32903, 32757, 32905] avg=32768.0000 max=33037 max_rank=1 min=32380 min_rank=0 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] avg=32768.0000 max=33096 max_rank=0 min=32627 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=49 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] avg=32768.0000 max=33096 max_rank=0 min=32627 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=49 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] avg=32768.0000 max=33096 max_rank=0 min=32627 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=49 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] avg=32768.0000 max=33096 max_rank=0 min=32627 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=49 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] avg=32768.0000 max=33096 max_rank=0 min=32627 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=49 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33121, 32996, 32823, 32651, 32673, 32734, 32506] avg=32768.0000 max=33121 max_rank=1 min=32506 min_rank=7 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33121, 32996, 32823, 32651, 32673, 32734, 32506] avg=32768.0000 max=33121 max_rank=1 min=32506 min_rank=7 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33121, 32996, 32823, 32651, 32673, 32734, 32506] avg=32768.0000 max=33121 max_rank=1 min=32506 min_rank=7 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33121, 32996, 32823, 32651, 32673, 32734, 32506] avg=32768.0000 max=33121 max_rank=1 min=32506 min_rank=7 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33121, 32996, 32823, 32651, 32673, 32734, 32506] avg=32768.0000 max=33121 max_rank=1 min=32506 min_rank=7 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] avg=32768.0000 max=33096 max_rank=1 min=32531 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] avg=32768.0000 max=33096 max_rank=1 min=32531 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32908, 32895, 32792, 32801, 32700, 32363, 32809, 32876] avg=32768.0000 max=32908 max_rank=0 min=32363 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] avg=32768.0000 max=33096 max_rank=1 min=32531 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] avg=32768.0000 max=33096 max_rank=1 min=32531 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] avg=32768.0000 max=33096 max_rank=1 min=32531 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[33126, 32683, 32678, 32597, 32624, 33115, 32659, 32662] avg=32768.0000 max=33126 max_rank=0 min=32597 min_rank=3 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[32940, 32457, 32521, 33078, 32643, 32665, 32877, 32963] avg=32768.0000 max=33078 max_rank=3 min=32457 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[33126, 32683, 32678, 32597, 32624, 33115, 32659, 32662] avg=32768.0000 max=33126 max_rank=0 min=32597 min_rank=3 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] avg=32768.0000 max=33096 max_rank=0 min=32627 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=49 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] avg=32768.0000 max=33096 max_rank=0 min=32627 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=49 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[33126, 32683, 32678, 32597, 32624, 33115, 32659, 32662] avg=32768.0000 max=33126 max_rank=0 min=32597 min_rank=3 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33121, 32996, 32823, 32651, 32673, 32734, 32506] avg=32768.0000 max=33121 max_rank=1 min=32506 min_rank=7 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[33096, 32683, 32678, 32627, 32643, 33096, 32659, 32662] avg=32768.0000 max=33096 max_rank=0 min=32627 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=49 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33121, 32996, 32823, 32651, 32673, 32734, 32506] avg=32768.0000 max=33121 max_rank=1 min=32506 min_rank=7 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] avg=32768.0000 max=33096 max_rank=1 min=32531 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] avg=32768.0000 max=33096 max_rank=1 min=32531 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33121, 32996, 32823, 32651, 32673, 32734, 32506] avg=32768.0000 max=33121 max_rank=1 min=32506 min_rank=7 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[32640, 33096, 32996, 32823, 32651, 32673, 32734, 32531] avg=32768.0000 max=33096 max_rank=1 min=32531 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[32775, 32762, 32800, 32663, 32757, 32837, 32815, 32735] avg=32768.0000 max=32837 max_rank=5 min=32663 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[32759, 32819, 32737, 32933, 32807, 32663, 32655, 32771] avg=32768.0000 max=32933 max_rank=3 min=32655 min_rank=6 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[32729, 32634, 32710, 33058, 32988, 32768, 32806, 32451] avg=32768.0000 max=33058 max_rank=3 min=32451 min_rank=7 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[32598, 33038, 32717, 32925, 33021, 32495, 32670, 32680] avg=32768.0000 max=33038 max_rank=1 min=32495 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[32701, 32771, 32769, 32706, 32762, 32664, 32973, 32798] avg=32768.0000 max=32973 max_rank=6 min=32664 min_rank=5 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[32913, 32766, 32797, 32673, 32724, 32775, 32590, 32906] avg=32768.0000 max=32913 max_rank=0 min=32590 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=25 tokens_per_ep_rank=[32797, 32852, 32664, 32771, 32839, 32708, 32694, 32819] avg=32768.0000 max=32852 max_rank=1 min=32664 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33123, 32350, 32835] avg=32768.0000 max=33123 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33123, 32350, 32835] avg=32768.0000 max=33123 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33123, 32350, 32835] avg=32768.0000 max=33123 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=26 tokens_per_ep_rank=[32444, 32703, 32817, 32814, 32835, 32952, 32744, 32835] avg=32768.0000 max=32952 max_rank=5 min=32444 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] avg=32768.0000 max=33096 max_rank=5 min=32377 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000815 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] avg=32768.0000 max=33096 max_rank=5 min=32377 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000815 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33123, 32350, 32835] avg=32768.0000 max=33123 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] avg=32768.0000 max=33096 max_rank=5 min=32377 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000815 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33123, 32350, 32835] avg=32768.0000 max=33123 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] avg=32768.0000 max=33096 max_rank=5 min=32377 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000815 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] avg=32768.0000 max=33096 max_rank=5 min=32377 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000815 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=27 tokens_per_ep_rank=[32912, 32971, 32618, 32493, 32787, 32830, 32794, 32739] avg=32768.0000 max=32971 max_rank=1 min=32493 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=28 tokens_per_ep_rank=[32964, 32694, 32982, 32698, 32573, 32568, 33047, 32618] avg=32768.0000 max=33047 max_rank=6 min=32568 min_rank=5 imbalance_ratio=1.008514 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33123, 32350, 32835] avg=32768.0000 max=33123 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=29 tokens_per_ep_rank=[32702, 32613, 32840, 33056, 32634, 32926, 32755, 32618] avg=32768.0000 max=33056 max_rank=3 min=32613 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] avg=32768.0000 max=33096 max_rank=5 min=32377 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000815 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33123, 32350, 32835] avg=32768.0000 max=33123 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33123, 32350, 32835] avg=32768.0000 max=33123 max_rank=5 min=32350 min_rank=6 imbalance_ratio=1.010834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] avg=32768.0000 max=33096 max_rank=5 min=32377 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000815 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=30 tokens_per_ep_rank=[33063, 32590, 32964, 32480, 32739, 33096, 32377, 32835] avg=32768.0000 max=33096 max_rank=5 min=32377 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000815 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=31 tokens_per_ep_rank=[32773, 32806, 32602, 32869, 32918, 32683, 32848, 32645] avg=32768.0000 max=32918 max_rank=4 min=32602 min_rank=2 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=32 tokens_per_ep_rank=[32872, 32868, 32890, 32880, 32739, 32720, 32669, 32506] avg=32768.0000 max=32890 max_rank=2 min=32506 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33140, 32661, 32688, 32752, 32528] avg=32768.0000 max=33140 max_rank=3 min=32528 min_rank=7 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33140, 32661, 32688, 32752, 32528] avg=32768.0000 max=33140 max_rank=3 min=32528 min_rank=7 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33140, 32661, 32688, 32752, 32528] avg=32768.0000 max=33140 max_rank=3 min=32528 min_rank=7 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33140, 32661, 32688, 32752, 32528] avg=32768.0000 max=33140 max_rank=3 min=32528 min_rank=7 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] avg=32768.0000 max=33096 max_rank=3 min=32572 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] avg=32768.0000 max=33096 max_rank=3 min=32572 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] avg=32768.0000 max=33096 max_rank=3 min=32572 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=33 tokens_per_ep_rank=[32888, 32679, 32859, 32939, 32673, 32505, 32833, 32768] avg=32768.0000 max=32939 max_rank=3 min=32505 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] avg=32768.0000 max=33096 max_rank=3 min=32572 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33140, 32661, 32688, 32752, 32528] avg=32768.0000 max=33140 max_rank=3 min=32528 min_rank=7 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] avg=32768.0000 max=33096 max_rank=3 min=32572 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=34 tokens_per_ep_rank=[32759, 32611, 32798, 33096, 32546, 32735, 32928, 32671] avg=32768.0000 max=33096 max_rank=3 min=32546 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=35 tokens_per_ep_rank=[32880, 32694, 32779, 32630, 32830, 32639, 32857, 32835] avg=32768.0000 max=32880 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.003418 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33140, 32661, 32688, 32752, 32528] avg=32768.0000 max=33140 max_rank=3 min=32528 min_rank=7 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=36 tokens_per_ep_rank=[32826, 32501, 32407, 32854, 32899, 32954, 32636, 33067] avg=32768.0000 max=33067 max_rank=7 min=32407 min_rank=2 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33140, 32661, 32688, 32752, 32528] avg=32768.0000 max=33140 max_rank=3 min=32528 min_rank=7 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] avg=32768.0000 max=33096 max_rank=3 min=32572 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33269, 32524, 32822, 32834, 32848, 32626] avg=32768.0000 max=33269 max_rank=2 min=32524 min_rank=3 imbalance_ratio=1.015289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33269, 32524, 32822, 32834, 32848, 32626] avg=32768.0000 max=33269 max_rank=2 min=32524 min_rank=3 imbalance_ratio=1.015289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] avg=32768.0000 max=33096 max_rank=3 min=32572 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=173 max_load_reduction_pct=0.005200 moved_to_replicas=173 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33269, 32524, 32822, 32834, 32848, 32626] avg=32768.0000 max=33269 max_rank=2 min=32524 min_rank=3 imbalance_ratio=1.015289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33140, 32661, 32688, 32752, 32528] avg=32768.0000 max=33140 max_rank=3 min=32528 min_rank=7 imbalance_ratio=1.011353 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=173 max_load_reduction_pct=0.005200 moved_to_replicas=173 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33269, 32524, 32822, 32834, 32848, 32626] avg=32768.0000 max=33269 max_rank=2 min=32524 min_rank=3 imbalance_ratio=1.015289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=173 max_load_reduction_pct=0.005200 moved_to_replicas=173 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=37 tokens_per_ep_rank=[32698, 32859, 32818, 33096, 32661, 32688, 32752, 32572] avg=32768.0000 max=33096 max_rank=3 min=32572 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.001328 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=173 max_load_reduction_pct=0.005200 moved_to_replicas=173 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33269, 32524, 32822, 32834, 32848, 32626] avg=32768.0000 max=33269 max_rank=2 min=32524 min_rank=3 imbalance_ratio=1.015289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=173 max_load_reduction_pct=0.005200 moved_to_replicas=173 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=38 tokens_per_ep_rank=[32563, 32673, 32658, 33059, 32826, 32512, 32821, 33032] avg=32768.0000 max=33059 max_rank=3 min=32512 min_rank=5 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=39 tokens_per_ep_rank=[32898, 32550, 32607, 32792, 32797, 32951, 32704, 32845] avg=32768.0000 max=32951 max_rank=5 min=32550 min_rank=1 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33269, 32524, 32822, 32834, 32848, 32626] avg=32768.0000 max=33269 max_rank=2 min=32524 min_rank=3 imbalance_ratio=1.015289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33324, 32565, 32747, 32628] avg=32768.0000 max=33324 max_rank=4 min=32565 min_rank=5 imbalance_ratio=1.016968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=40 tokens_per_ep_rank=[32646, 33079, 32809, 32621, 32864, 32497, 32818, 32810] avg=32768.0000 max=33079 max_rank=1 min=32497 min_rank=5 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33269, 32524, 32822, 32834, 32848, 32626] avg=32768.0000 max=33269 max_rank=2 min=32524 min_rank=3 imbalance_ratio=1.015289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=173 max_load_reduction_pct=0.005200 moved_to_replicas=173 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33324, 32565, 32747, 32628] avg=32768.0000 max=33324 max_rank=4 min=32565 min_rank=5 imbalance_ratio=1.016968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] avg=32768.0000 max=33096 max_rank=4 min=32588 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=228 max_load_reduction_pct=0.006842 moved_to_replicas=228 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33324, 32565, 32747, 32628] avg=32768.0000 max=33324 max_rank=4 min=32565 min_rank=5 imbalance_ratio=1.016968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=173 max_load_reduction_pct=0.005200 moved_to_replicas=173 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] avg=32768.0000 max=33096 max_rank=4 min=32588 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=228 max_load_reduction_pct=0.006842 moved_to_replicas=228 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33269, 32524, 32822, 32834, 32848, 32626] avg=32768.0000 max=33269 max_rank=2 min=32524 min_rank=3 imbalance_ratio=1.015289 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33324, 32565, 32747, 32628] avg=32768.0000 max=33324 max_rank=4 min=32565 min_rank=5 imbalance_ratio=1.016968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] avg=32768.0000 max=33096 max_rank=4 min=32588 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=228 max_load_reduction_pct=0.006842 moved_to_replicas=228 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=41 tokens_per_ep_rank=[32533, 32688, 33096, 32697, 32822, 32834, 32848, 32626] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=173 max_load_reduction_pct=0.005200 moved_to_replicas=173 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] avg=32768.0000 max=33096 max_rank=4 min=32588 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=228 max_load_reduction_pct=0.006842 moved_to_replicas=228 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33324, 32565, 32747, 32628] avg=32768.0000 max=33324 max_rank=4 min=32565 min_rank=5 imbalance_ratio=1.016968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] avg=32768.0000 max=33096 max_rank=4 min=32588 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=228 max_load_reduction_pct=0.006842 moved_to_replicas=228 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=42 tokens_per_ep_rank=[33080, 32625, 32758, 32627, 32784, 32687, 32913, 32670] avg=32768.0000 max=33080 max_rank=0 min=32625 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=43 tokens_per_ep_rank=[32793, 32666, 32905, 33008, 32892, 32634, 32623, 32623] avg=32768.0000 max=33008 max_rank=3 min=32623 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33324, 32565, 32747, 32628] avg=32768.0000 max=33324 max_rank=4 min=32565 min_rank=5 imbalance_ratio=1.016968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=44 tokens_per_ep_rank=[32837, 32770, 32573, 32892, 32875, 32500, 32913, 32784] avg=32768.0000 max=32913 max_rank=6 min=32500 min_rank=5 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] avg=32768.0000 max=33096 max_rank=4 min=32588 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=228 max_load_reduction_pct=0.006842 moved_to_replicas=228 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33324, 32565, 32747, 32628] avg=32768.0000 max=33324 max_rank=4 min=32565 min_rank=5 imbalance_ratio=1.016968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] avg=32768.0000 max=33096 max_rank=4 min=32588 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=228 max_load_reduction_pct=0.006842 moved_to_replicas=228 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33324, 32565, 32747, 32628] avg=32768.0000 max=33324 max_rank=4 min=32565 min_rank=5 imbalance_ratio=1.016968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=45 tokens_per_ep_rank=[32588, 32682, 32789, 32821, 33096, 32793, 32747, 32628] avg=32768.0000 max=33096 max_rank=4 min=32588 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=228 max_load_reduction_pct=0.006842 moved_to_replicas=228 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=46 tokens_per_ep_rank=[32840, 32544, 32736, 32948, 32668, 32649, 33083, 32676] avg=32768.0000 max=33083 max_rank=6 min=32544 min_rank=1 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=47 tokens_per_ep_rank=[32894, 32604, 32805, 32699, 32683, 32909, 32664, 32886] avg=32768.0000 max=32909 max_rank=5 min=32604 min_rank=1 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=48 tokens_per_ep_rank=[32734, 32505, 32931, 32802, 32864, 32467, 32892, 32949] avg=32768.0000 max=32949 max_rank=7 min=32467 min_rank=5 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=48 total_expected_recv=1571020 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=347264 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=48 total_expected_recv=1574141 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342016 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=48 total_expected_recv=1572898 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343424 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=48 total_expected_recv=1572655 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344960 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=48 total_expected_recv=1574488 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341504 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=48 total_expected_recv=1570359 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=348928 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=48 total_expected_recv=1574343 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342656 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=48 total_expected_recv=1573008 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344064 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:48:20] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 904.1 | throughput per GPU (TFLOP/s/GPU): 98.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.909838E+00 | loss scale: 1.0 | grad norm: 3.383 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32601 expected_overflow=0 exact_recv_per_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32601 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32462 expected_overflow=0 exact_recv_per_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32462 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33037 expected_overflow=0 exact_recv_per_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33037 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32798 expected_overflow=0 exact_recv_per_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32624 expected_overflow=0 exact_recv_per_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32798 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32624 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32495 expected_overflow=0 exact_recv_per_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33003 expected_overflow=0 exact_recv_per_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32495 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=33003 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32559 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33046 expected_overflow=0 exact_recv_per_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=40960 expected_recv=33046 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32448 expected_overflow=0 exact_recv_per_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32448 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32548 expected_overflow=0 exact_recv_per_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32548 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32570 expected_overflow=0 exact_recv_per_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32570 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33052 expected_overflow=0 exact_recv_per_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=40960 expected_recv=33052 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32570 expected_overflow=0 exact_recv_per_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32570 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32391 expected_overflow=0 exact_recv_per_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32391 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32938 expected_overflow=0 exact_recv_per_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32938 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32507 expected_overflow=0 exact_recv_per_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32507 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=23 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32524 expected_overflow=0 exact_recv_per_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32524 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32595 expected_overflow=0 exact_recv_per_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32595 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=25 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=24 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=25 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33042 expected_overflow=0 exact_recv_per_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33042 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=28 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32834 expected_overflow=0 exact_recv_per_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=27 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=26 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32834 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32544 expected_overflow=0 exact_recv_per_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32544 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=27 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33020 expected_overflow=0 exact_recv_per_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=30 calls=1 budget=40960 expected_recv=33020 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=28 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33004 expected_overflow=0 exact_recv_per_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=31 calls=1 budget=40960 expected_recv=33004 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32570 expected_overflow=0 exact_recv_per_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32570 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=29 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32601 expected_overflow=0 exact_recv_per_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=30 calls=1 budget=40960 expected_recv=32601 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32511 expected_overflow=0 exact_recv_per_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=32 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=31 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32511 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33086 expected_overflow=0 exact_recv_per_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=32 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33086 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32595 expected_overflow=0 exact_recv_per_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32595 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32998 expected_overflow=0 exact_recv_per_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=34 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=33 calls=1 budget=40960 expected_recv=32998 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=34 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32452 expected_overflow=0 exact_recv_per_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=36 calls=1 budget=40960 expected_recv=32452 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=35 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33059 expected_overflow=0 exact_recv_per_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33014 expected_overflow=0 exact_recv_per_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=40 calls=1 budget=40960 expected_recv=33059 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=36 calls=1 budget=40960 expected_recv=33014 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32363 expected_overflow=0 exact_recv_per_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32363 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=37 calls=1 budget=40960 expected_recv=32393 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32983 expected_overflow=0 exact_recv_per_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32983 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=38 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32974 expected_overflow=0 exact_recv_per_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=39 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32974 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32593 expected_overflow=0 exact_recv_per_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32593 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=40 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32936 expected_overflow=0 exact_recv_per_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=41 calls=1 budget=40960 expected_recv=32936 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=46 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=44 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=42 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32978 expected_overflow=0 exact_recv_per_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=43 calls=1 budget=40960 expected_recv=32978 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32938 expected_overflow=0 exact_recv_per_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32938 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32619 expected_overflow=0 exact_recv_per_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=44 calls=1 budget=40960 expected_recv=32619 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32543 expected_overflow=0 exact_recv_per_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32543 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=45 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=46 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=47 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32764, 32851, 32713, 32601, 32599, 32808, 32988, 32820] avg=32768.0000 max=32988 max_rank=6 min=32599 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32628, 32666, 33037, 32791, 32853, 32462, 32976, 32731] avg=32768.0000 max=33037 max_rank=2 min=32462 min_rank=5 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32798, 32854, 32825, 32864, 32624, 32675, 32592, 32912] avg=32768.0000 max=32912 max_rank=7 min=32592 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[33003, 32650, 32856, 32887, 32661, 32568, 33024, 32495] avg=32768.0000 max=33024 max_rank=6 min=32495 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33226, 32832, 32833, 32457, 32805, 32736, 32559, 32696] avg=32768.0000 max=33226 max_rank=0 min=32457 min_rank=3 imbalance_ratio=1.013977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32894, 32718, 32879, 32816, 32728, 32677, 32717, 32715] avg=32768.0000 max=32894 max_rank=0 min=32677 min_rank=5 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] avg=32768.0000 max=33096 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130 max_load_reduction_pct=0.003913 moved_to_replicas=130 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33226, 32832, 32833, 32457, 32805, 32736, 32559, 32696] avg=32768.0000 max=33226 max_rank=0 min=32457 min_rank=3 imbalance_ratio=1.013977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] avg=32768.0000 max=33096 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130 max_load_reduction_pct=0.003913 moved_to_replicas=130 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33226, 32832, 32833, 32457, 32805, 32736, 32559, 32696] avg=32768.0000 max=33226 max_rank=0 min=32457 min_rank=3 imbalance_ratio=1.013977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32659, 32807, 32765, 32857, 32597, 32748, 32858, 32853] avg=32768.0000 max=32858 max_rank=6 min=32597 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33226, 32832, 32833, 32457, 32805, 32736, 32559, 32696] avg=32768.0000 max=33226 max_rank=0 min=32457 min_rank=3 imbalance_ratio=1.013977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] avg=32768.0000 max=33096 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130 max_load_reduction_pct=0.003913 moved_to_replicas=130 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33226, 32832, 32833, 32457, 32805, 32736, 32559, 32696] avg=32768.0000 max=33226 max_rank=0 min=32457 min_rank=3 imbalance_ratio=1.013977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] avg=32768.0000 max=33096 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130 max_load_reduction_pct=0.003913 moved_to_replicas=130 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] avg=32768.0000 max=33096 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130 max_load_reduction_pct=0.003913 moved_to_replicas=130 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32561, 32860, 32636, 32980, 32672, 32954, 32807, 32674] avg=32768.0000 max=32980 max_rank=3 min=32561 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33226, 32832, 32833, 32457, 32805, 32736, 32559, 32696] avg=32768.0000 max=33226 max_rank=0 min=32457 min_rank=3 imbalance_ratio=1.013977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] avg=32768.0000 max=33096 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130 max_load_reduction_pct=0.003913 moved_to_replicas=130 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33226, 32832, 32833, 32457, 32805, 32736, 32559, 32696] avg=32768.0000 max=33226 max_rank=0 min=32457 min_rank=3 imbalance_ratio=1.013977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32956, 32665, 32985, 32763, 32837, 32564, 32575, 32799] avg=32768.0000 max=32985 max_rank=2 min=32564 min_rank=5 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] avg=32768.0000 max=33096 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130 max_load_reduction_pct=0.003913 moved_to_replicas=130 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33006, 32533, 32748, 32672, 32581, 32735, 32979, 32890] avg=32768.0000 max=33006 max_rank=0 min=32533 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33226, 32832, 32833, 32457, 32805, 32736, 32559, 32696] avg=32768.0000 max=33226 max_rank=0 min=32457 min_rank=3 imbalance_ratio=1.013977 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32832, 32833, 32587, 32805, 32736, 32559, 32696] avg=32768.0000 max=33096 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=130 max_load_reduction_pct=0.003913 moved_to_replicas=130 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[32757, 32448, 32747, 32812, 32611, 33046, 32951, 32772] avg=32768.0000 max=33046 max_rank=5 min=32448 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[32627, 32904, 32703, 32776, 32757, 32843, 32986, 32548] avg=32768.0000 max=32986 max_rank=6 min=32548 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32786, 32932, 32570, 32930, 32556, 32572, 32940, 32858] avg=32768.0000 max=32940 max_rank=6 min=32556 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32904, 32560, 32894, 32775, 32837, 32792, 32698, 32684] avg=32768.0000 max=32904 max_rank=0 min=32560 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32820, 32618, 32594, 32893, 32986, 32654, 32649, 32930] avg=32768.0000 max=32986 max_rank=4 min=32594 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[32912, 32853, 32656, 32727, 32951, 32572, 32579, 32894] avg=32768.0000 max=32951 max_rank=4 min=32572 min_rank=5 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[32980, 32577, 32924, 32526, 32870, 32763, 32857, 32647] avg=32768.0000 max=32980 max_rank=0 min=32526 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33164, 32439, 32729, 32631, 32875] avg=32768.0000 max=33164 max_rank=3 min=32439 min_rank=4 imbalance_ratio=1.012085 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[32723, 32529, 33052, 32930, 32822, 32586, 32774, 32728] avg=32768.0000 max=33052 max_rank=2 min=32529 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=68 max_load_reduction_pct=0.002050 moved_to_replicas=68 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33164, 32439, 32729, 32631, 32875] avg=32768.0000 max=33164 max_rank=3 min=32439 min_rank=4 imbalance_ratio=1.012085 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[32739, 32722, 32526, 32627, 32937, 33053, 32856, 32684] avg=32768.0000 max=33053 max_rank=5 min=32526 min_rank=2 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=68 max_load_reduction_pct=0.002050 moved_to_replicas=68 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33164, 32439, 32729, 32631, 32875] avg=32768.0000 max=33164 max_rank=3 min=32439 min_rank=4 imbalance_ratio=1.012085 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33164, 32439, 32729, 32631, 32875] avg=32768.0000 max=33164 max_rank=3 min=32439 min_rank=4 imbalance_ratio=1.012085 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=68 max_load_reduction_pct=0.002050 moved_to_replicas=68 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=68 max_load_reduction_pct=0.002050 moved_to_replicas=68 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[32725, 32761, 32737, 32931, 32625, 32733, 32736, 32896] avg=32768.0000 max=32931 max_rank=3 min=32625 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33164, 32439, 32729, 32631, 32875] avg=32768.0000 max=33164 max_rank=3 min=32439 min_rank=4 imbalance_ratio=1.012085 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=68 max_load_reduction_pct=0.002050 moved_to_replicas=68 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[32753, 32966, 32761, 32828, 32972, 32570, 32605, 32689] avg=32768.0000 max=32972 max_rank=4 min=32570 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33164, 32439, 32729, 32631, 32875] avg=32768.0000 max=33164 max_rank=3 min=32439 min_rank=4 imbalance_ratio=1.012085 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=68 max_load_reduction_pct=0.002050 moved_to_replicas=68 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33164, 32439, 32729, 32631, 32875] avg=32768.0000 max=33164 max_rank=3 min=32439 min_rank=4 imbalance_ratio=1.012085 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[32711, 32877, 32828, 32662, 32391, 32865, 32830, 32980] avg=32768.0000 max=32980 max_rank=7 min=32391 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=68 max_load_reduction_pct=0.002050 moved_to_replicas=68 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[32668, 32727, 32938, 32780, 32505, 32871, 32880, 32775] avg=32768.0000 max=32938 max_rank=2 min=32505 min_rank=4 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33164, 32439, 32729, 32631, 32875] avg=32768.0000 max=33164 max_rank=3 min=32439 min_rank=4 imbalance_ratio=1.012085 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[32648, 32844, 32814, 33096, 32507, 32729, 32631, 32875] avg=32768.0000 max=33096 max_rank=3 min=32507 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=68 max_load_reduction_pct=0.002050 moved_to_replicas=68 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=25 tokens_per_ep_rank=[32837, 32595, 32810, 32647, 32726, 32746, 32893, 32890] avg=32768.0000 max=32893 max_rank=6 min=32595 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=26 tokens_per_ep_rank=[32707, 32765, 33035, 32722, 33023, 32730, 32638, 32524] avg=32768.0000 max=33035 max_rank=2 min=32524 min_rank=7 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=27 tokens_per_ep_rank=[32962, 32467, 32822, 32801, 32699, 32931, 32827, 32635] avg=32768.0000 max=32962 max_rank=0 min=32467 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=28 tokens_per_ep_rank=[32834, 33010, 32882, 32707, 32628, 32786, 32720, 32577] avg=32768.0000 max=33010 max_rank=1 min=32577 min_rank=7 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=29 tokens_per_ep_rank=[32598, 32774, 32518, 32680, 32789, 33025, 32718, 33042] avg=32768.0000 max=33042 max_rank=7 min=32518 min_rank=2 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=30 tokens_per_ep_rank=[32859, 32673, 32881, 32824, 32839, 32652, 32872, 32544] avg=32768.0000 max=32881 max_rank=2 min=32544 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=31 tokens_per_ep_rank=[32891, 32570, 32805, 32601, 32966, 33020, 32532, 32759] avg=32768.0000 max=33020 max_rank=5 min=32532 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=32 tokens_per_ep_rank=[32839, 32780, 32815, 32655, 32838, 32585, 32628, 33004] avg=32768.0000 max=33004 max_rank=7 min=32585 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=33 tokens_per_ep_rank=[32535, 32743, 32693, 33006, 32854, 32610, 32896, 32807] avg=32768.0000 max=33006 max_rank=3 min=32535 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=34 tokens_per_ep_rank=[32591, 32525, 32683, 32998, 32895, 32635, 32906, 32911] avg=32768.0000 max=32998 max_rank=3 min=32525 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=35 tokens_per_ep_rank=[32740, 32574, 33086, 33018, 32790, 32482, 32943, 32511] avg=32768.0000 max=33086 max_rank=2 min=32482 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=36 tokens_per_ep_rank=[32616, 32815, 32650, 32953, 32734, 32797, 32833, 32746] avg=32768.0000 max=32953 max_rank=3 min=32616 min_rank=0 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=37 tokens_per_ep_rank=[32452, 32877, 32750, 33014, 32897, 32861, 32595, 32698] avg=32768.0000 max=33014 max_rank=3 min=32452 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33321, 32516, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33321 max_rank=1 min=32516 min_rank=2 imbalance_ratio=1.016876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=38 tokens_per_ep_rank=[32976, 32913, 32772, 32393, 32658, 32994, 32661, 32777] avg=32768.0000 max=32994 max_rank=5 min=32393 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33096 max_rank=1 min=32585 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=225 max_load_reduction_pct=0.006752 moved_to_replicas=225 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32549, 32799, 32739, 33141, 32634] avg=32768.0000 max=33141 max_rank=6 min=32549 min_rank=3 imbalance_ratio=1.011383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33321, 32516, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33321 max_rank=1 min=32516 min_rank=2 imbalance_ratio=1.016876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=39 tokens_per_ep_rank=[32939, 32667, 32710, 32769, 32532, 32786, 32909, 32832] avg=32768.0000 max=32939 max_rank=0 min=32532 min_rank=4 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] avg=32768.0000 max=33096 max_rank=6 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45 max_load_reduction_pct=0.001358 moved_to_replicas=45 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33096 max_rank=1 min=32585 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=225 max_load_reduction_pct=0.006752 moved_to_replicas=225 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33321, 32516, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33321 max_rank=1 min=32516 min_rank=2 imbalance_ratio=1.016876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33321, 32516, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33321 max_rank=1 min=32516 min_rank=2 imbalance_ratio=1.016876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=47 tokens_per_ep_rank=[32512, 32757, 32558, 32787, 32698, 32985, 33127, 32720] avg=32768.0000 max=33127 max_rank=6 min=32512 min_rank=0 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33096 max_rank=1 min=32585 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=225 max_load_reduction_pct=0.006752 moved_to_replicas=225 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32549, 32799, 32739, 33141, 32634] avg=32768.0000 max=33141 max_rank=6 min=32549 min_rank=3 imbalance_ratio=1.011383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33096 max_rank=1 min=32585 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=225 max_load_reduction_pct=0.006752 moved_to_replicas=225 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=47 tokens_per_ep_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] avg=32768.0000 max=33096 max_rank=6 min=32543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=40 tokens_per_ep_rank=[32929, 32750, 32865, 32775, 32710, 32649, 32904, 32562] avg=32768.0000 max=32929 max_rank=0 min=32562 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] avg=32768.0000 max=33096 max_rank=6 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45 max_load_reduction_pct=0.001358 moved_to_replicas=45 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32549, 32799, 32739, 33141, 32634] avg=32768.0000 max=33141 max_rank=6 min=32549 min_rank=3 imbalance_ratio=1.011383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32549, 32799, 32739, 33141, 32634] avg=32768.0000 max=33141 max_rank=6 min=32549 min_rank=3 imbalance_ratio=1.011383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33321, 32516, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33321 max_rank=1 min=32516 min_rank=2 imbalance_ratio=1.016876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] avg=32768.0000 max=33096 max_rank=6 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45 max_load_reduction_pct=0.001358 moved_to_replicas=45 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=47 tokens_per_ep_rank=[32512, 32757, 32558, 32787, 32698, 32985, 33127, 32720] avg=32768.0000 max=33127 max_rank=6 min=32512 min_rank=0 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] avg=32768.0000 max=33096 max_rank=6 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45 max_load_reduction_pct=0.001358 moved_to_replicas=45 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33096 max_rank=1 min=32585 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=225 max_load_reduction_pct=0.006752 moved_to_replicas=225 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=41 tokens_per_ep_rank=[32743, 32906, 32983, 32613, 32807, 32670, 33059, 32363] avg=32768.0000 max=33059 max_rank=6 min=32363 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=47 tokens_per_ep_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] avg=32768.0000 max=33096 max_rank=6 min=32543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=47 tokens_per_ep_rank=[32512, 32757, 32558, 32787, 32698, 32985, 33127, 32720] avg=32768.0000 max=33127 max_rank=6 min=32512 min_rank=0 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=47 tokens_per_ep_rank=[32512, 32757, 32558, 32787, 32698, 32985, 33127, 32720] avg=32768.0000 max=33127 max_rank=6 min=32512 min_rank=0 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32549, 32799, 32739, 33141, 32634] avg=32768.0000 max=33141 max_rank=6 min=32549 min_rank=3 imbalance_ratio=1.011383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=47 tokens_per_ep_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] avg=32768.0000 max=33096 max_rank=6 min=32543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=47 tokens_per_ep_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] avg=32768.0000 max=33096 max_rank=6 min=32543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] avg=32768.0000 max=33096 max_rank=6 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45 max_load_reduction_pct=0.001358 moved_to_replicas=45 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33321, 32516, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33321 max_rank=1 min=32516 min_rank=2 imbalance_ratio=1.016876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=42 tokens_per_ep_rank=[32670, 32724, 32794, 32936, 32745, 32845, 32717, 32713] avg=32768.0000 max=32936 max_rank=3 min=32670 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33096 max_rank=1 min=32585 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=225 max_load_reduction_pct=0.006752 moved_to_replicas=225 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=47 tokens_per_ep_rank=[32512, 32757, 32558, 32787, 32698, 32985, 33127, 32720] avg=32768.0000 max=33127 max_rank=6 min=32512 min_rank=0 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=47 tokens_per_ep_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] avg=32768.0000 max=33096 max_rank=6 min=32543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32549, 32799, 32739, 33141, 32634] avg=32768.0000 max=33141 max_rank=6 min=32549 min_rank=3 imbalance_ratio=1.011383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33321, 32516, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33321 max_rank=1 min=32516 min_rank=2 imbalance_ratio=1.016876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=43 tokens_per_ep_rank=[32858, 32970, 32974, 32776, 32593, 32665, 32433, 32875] avg=32768.0000 max=32974 max_rank=2 min=32433 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] avg=32768.0000 max=33096 max_rank=6 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45 max_load_reduction_pct=0.001358 moved_to_replicas=45 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33096 max_rank=1 min=32585 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=225 max_load_reduction_pct=0.006752 moved_to_replicas=225 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=47 tokens_per_ep_rank=[32512, 32757, 32558, 32787, 32698, 32985, 33127, 32720] avg=32768.0000 max=33127 max_rank=6 min=32512 min_rank=0 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32549, 32799, 32739, 33141, 32634] avg=32768.0000 max=33141 max_rank=6 min=32549 min_rank=3 imbalance_ratio=1.011383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=44 tokens_per_ep_rank=[32732, 32523, 32558, 32978, 32898, 32802, 32967, 32686] avg=32768.0000 max=32978 max_rank=3 min=32523 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=47 tokens_per_ep_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] avg=32768.0000 max=33096 max_rank=6 min=32543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] avg=32768.0000 max=33096 max_rank=6 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45 max_load_reduction_pct=0.001358 moved_to_replicas=45 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33321, 32516, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33321 max_rank=1 min=32516 min_rank=2 imbalance_ratio=1.016876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=47 tokens_per_ep_rank=[32512, 32757, 32558, 32787, 32698, 32985, 33127, 32720] avg=32768.0000 max=33127 max_rank=6 min=32512 min_rank=0 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=45 tokens_per_ep_rank=[32585, 33096, 32741, 32619, 32689, 32745, 32815, 32854] avg=32768.0000 max=33096 max_rank=1 min=32585 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=225 max_load_reduction_pct=0.006752 moved_to_replicas=225 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=47 tokens_per_ep_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] avg=32768.0000 max=33096 max_rank=6 min=32543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32549, 32799, 32739, 33141, 32634] avg=32768.0000 max=33141 max_rank=6 min=32549 min_rank=3 imbalance_ratio=1.011383 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=46 tokens_per_ep_rank=[32633, 32805, 32844, 32594, 32799, 32739, 33096, 32634] avg=32768.0000 max=33096 max_rank=6 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45 max_load_reduction_pct=0.001358 moved_to_replicas=45 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=47 tokens_per_ep_rank=[32512, 32757, 32558, 32787, 32698, 32985, 33127, 32720] avg=32768.0000 max=33127 max_rank=6 min=32512 min_rank=0 imbalance_ratio=1.010956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=47 tokens_per_ep_rank=[32543, 32757, 32558, 32787, 32698, 32985, 33096, 32720] avg=32768.0000 max=33096 max_rank=6 min=32543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31 max_load_reduction_pct=0.000936 moved_to_replicas=31 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=48 tokens_per_ep_rank=[32742, 32625, 32859, 32672, 32801, 32938, 32603, 32904] avg=32768.0000 max=32938 max_rank=5 min=32603 min_rank=6 imbalance_ratio=1.005188 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=48 total_expected_recv=1574308 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342912 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=48 total_expected_recv=1572113 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344832 peak_demand=33042 required_capacity=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=48 total_expected_recv=1572254 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=347264 peak_demand=33053 required_capacity=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=48 total_expected_recv=1574109 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342400 peak_demand=33086 required_capacity=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=48 total_expected_recv=1571892 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345856 peak_demand=33023 required_capacity=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=48 total_expected_recv=1571658 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344320 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=48 total_expected_recv=1572929 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344960 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=48 total_expected_recv=1573649 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343296 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:48:21] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 720.6 | throughput per GPU (TFLOP/s/GPU): 123.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.846426E+00 | loss scale: 1.0 | grad norm: 3.222 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32559 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33089 expected_overflow=0 exact_recv_per_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33089 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32543 expected_overflow=0 exact_recv_per_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32543 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32380 expected_overflow=0 exact_recv_per_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32380 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32936 expected_overflow=0 exact_recv_per_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32303 expected_overflow=0 exact_recv_per_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32960 expected_overflow=0 exact_recv_per_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32936 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32303 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32960 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32554 expected_overflow=0 exact_recv_per_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33062 expected_overflow=0 exact_recv_per_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32554 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33062 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32345 expected_overflow=0 exact_recv_per_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32345 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32515 expected_overflow=0 exact_recv_per_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32957 expected_overflow=0 exact_recv_per_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32515 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32957 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33013 expected_overflow=0 exact_recv_per_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=33013 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32605 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32409 expected_overflow=0 exact_recv_per_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32409 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33019 expected_overflow=0 exact_recv_per_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32517 expected_overflow=0 exact_recv_per_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32626 expected_overflow=0 exact_recv_per_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=40960 expected_recv=33019 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32517 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32626 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=40960 expected_recv=32639 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33067 expected_overflow=0 exact_recv_per_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=40960 expected_recv=33067 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33066 expected_overflow=0 exact_recv_per_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=40960 expected_recv=33066 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33021 expected_overflow=0 exact_recv_per_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33085 expected_overflow=0 exact_recv_per_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=40960 expected_recv=33021 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33085 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33003 expected_overflow=0 exact_recv_per_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=40960 expected_recv=33003 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33041 expected_overflow=0 exact_recv_per_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33040 expected_overflow=0 exact_recv_per_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=40960 expected_recv=33041 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=40960 expected_recv=33040 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32422 expected_overflow=0 exact_recv_per_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=23 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32422 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=24 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32991 expected_overflow=0 exact_recv_per_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32991 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32941 expected_overflow=0 exact_recv_per_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32941 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=25 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32436 expected_overflow=0 exact_recv_per_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32436 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=26 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=27 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32961 expected_overflow=0 exact_recv_per_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=28 calls=1 budget=40960 expected_recv=32961 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33086 expected_overflow=0 exact_recv_per_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=28 calls=1 budget=40960 expected_recv=33086 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33003 expected_overflow=0 exact_recv_per_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=29 calls=1 budget=40960 expected_recv=33003 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32415 expected_overflow=0 exact_recv_per_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32415 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=29 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33062 expected_overflow=0 exact_recv_per_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=30 calls=1 budget=40960 expected_recv=33062 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32553 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=30 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=31 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=31 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33087 expected_overflow=0 exact_recv_per_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=32 calls=1 budget=40960 expected_recv=33087 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32520 expected_overflow=0 exact_recv_per_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32983 expected_overflow=0 exact_recv_per_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32520 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=32 calls=1 budget=40960 expected_recv=32983 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=33 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32578 expected_overflow=0 exact_recv_per_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=34 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32578 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=36 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32495 expected_overflow=0 exact_recv_per_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32495 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=35 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32962 expected_overflow=0 exact_recv_per_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=36 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32962 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32481 expected_overflow=0 exact_recv_per_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32481 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=37 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32960 expected_overflow=0 exact_recv_per_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32960 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=38 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33031 expected_overflow=0 exact_recv_per_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=40 calls=1 budget=40960 expected_recv=33031 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=39 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33077 expected_overflow=0 exact_recv_per_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=41 calls=1 budget=40960 expected_recv=33077 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=40 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32394 expected_overflow=0 exact_recv_per_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32394 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=41 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=43 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=42 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=43 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33025 expected_overflow=0 exact_recv_per_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=46 calls=1 budget=40960 expected_recv=33025 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32498 expected_overflow=0 exact_recv_per_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=44 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32498 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=47 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32490 expected_overflow=0 exact_recv_per_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=45 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32461 expected_overflow=0 exact_recv_per_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32945 expected_overflow=0 exact_recv_per_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32490 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32461 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32945 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=46 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32559 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32379, 33323, 32718, 32482, 32782] avg=32768.0000 max=33323 max_rank=4 min=32379 min_rank=3 imbalance_ratio=1.016937 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32379, 33323, 32718, 32482, 32782] avg=32768.0000 max=33323 max_rank=4 min=32379 min_rank=3 imbalance_ratio=1.016937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] avg=32768.0000 max=33096 max_rank=4 min=32482 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=227 max_load_reduction_pct=0.006812 moved_to_replicas=227 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=47 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32379, 33323, 32718, 32482, 32782] avg=32768.0000 max=33323 max_rank=4 min=32379 min_rank=3 imbalance_ratio=1.016937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] avg=32768.0000 max=33096 max_rank=4 min=32482 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=227 max_load_reduction_pct=0.006812 moved_to_replicas=227 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32379, 33323, 32718, 32482, 32782] avg=32768.0000 max=33323 max_rank=4 min=32379 min_rank=3 imbalance_ratio=1.016937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] avg=32768.0000 max=33096 max_rank=4 min=32482 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=227 max_load_reduction_pct=0.006812 moved_to_replicas=227 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] avg=32768.0000 max=33096 max_rank=4 min=32482 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=227 max_load_reduction_pct=0.006812 moved_to_replicas=227 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32796, 32772, 32952, 32729, 32559, 32870, 32573, 32893] avg=32768.0000 max=32952 max_rank=2 min=32559 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32379, 33323, 32718, 32482, 32782] avg=32768.0000 max=33323 max_rank=4 min=32379 min_rank=3 imbalance_ratio=1.016937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32379, 33323, 32718, 32482, 32782] avg=32768.0000 max=33323 max_rank=4 min=32379 min_rank=3 imbalance_ratio=1.016937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33109, 32500, 32855, 33126, 32672] avg=32768.0000 max=33126 max_rank=6 min=32500 min_rank=4 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33109, 32500, 32855, 33126, 32672] avg=32768.0000 max=33126 max_rank=6 min=32500 min_rank=4 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32379, 33323, 32718, 32482, 32782] avg=32768.0000 max=33323 max_rank=4 min=32379 min_rank=3 imbalance_ratio=1.016937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] avg=32768.0000 max=33096 max_rank=4 min=32482 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=227 max_load_reduction_pct=0.006812 moved_to_replicas=227 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] avg=32768.0000 max=33096 max_rank=4 min=32482 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=227 max_load_reduction_pct=0.006812 moved_to_replicas=227 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] avg=32768.0000 max=33096 max_rank=3 min=32543 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=43 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32379, 33323, 32718, 32482, 32782] avg=32768.0000 max=33323 max_rank=4 min=32379 min_rank=3 imbalance_ratio=1.016937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] avg=32768.0000 max=33096 max_rank=3 min=32543 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=43 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33109, 32500, 32855, 33126, 32672] avg=32768.0000 max=33126 max_rank=6 min=32500 min_rank=4 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33109, 32500, 32855, 33126, 32672] avg=32768.0000 max=33126 max_rank=6 min=32500 min_rank=4 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] avg=32768.0000 max=33096 max_rank=4 min=32482 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=227 max_load_reduction_pct=0.006812 moved_to_replicas=227 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33089, 32580, 32791, 32606, 33096, 32718, 32482, 32782] avg=32768.0000 max=33096 max_rank=4 min=32482 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=227 max_load_reduction_pct=0.006812 moved_to_replicas=227 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] avg=32768.0000 max=33096 max_rank=3 min=32543 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=43 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] avg=32768.0000 max=33096 max_rank=3 min=32543 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=43 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33109, 32500, 32855, 33126, 32672] avg=32768.0000 max=33126 max_rank=6 min=32500 min_rank=4 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32811, 32774, 32674, 32722, 32639, 32658, 32979, 32887] avg=32768.0000 max=32979 max_rank=6 min=32639 min_rank=4 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33109, 32500, 32855, 33126, 32672] avg=32768.0000 max=33126 max_rank=6 min=32500 min_rank=4 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] avg=32768.0000 max=33096 max_rank=3 min=32543 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=43 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33109, 32500, 32855, 33126, 32672] avg=32768.0000 max=33126 max_rank=6 min=32500 min_rank=4 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] avg=32768.0000 max=33096 max_rank=3 min=32543 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=43 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33109, 32500, 32855, 33126, 32672] avg=32768.0000 max=33126 max_rank=6 min=32500 min_rank=4 imbalance_ratio=1.010925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] avg=32768.0000 max=33096 max_rank=3 min=32543 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=43 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32555, 32746, 32581, 33096, 32543, 32855, 33096, 32672] avg=32768.0000 max=33096 max_rank=3 min=32543 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=30 max_load_reduction_pct=0.000906 moved_to_replicas=43 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32936, 32996, 33045, 32778, 32960, 32746, 32303, 32380] avg=32768.0000 max=33045 max_rank=2 min=32303 min_rank=6 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33152, 32484, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33152 max_rank=0 min=32484 min_rank=1 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32862, 32607, 32554, 32614, 32812, 33062, 32771, 32862] avg=32768.0000 max=33062 max_rank=5 min=32554 min_rank=2 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33152, 32484, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33152 max_rank=0 min=32484 min_rank=1 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33096 max_rank=0 min=32515 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33152, 32484, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33152 max_rank=0 min=32484 min_rank=1 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33096 max_rank=0 min=32515 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33096 max_rank=0 min=32515 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33152, 32484, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33152 max_rank=0 min=32484 min_rank=1 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32345, 32607, 33034, 32981, 32689, 32857, 32919, 32712] avg=32768.0000 max=33034 max_rank=2 min=32345 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33096 max_rank=0 min=32515 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33152, 32484, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33152 max_rank=0 min=32484 min_rank=1 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33152, 32484, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33152 max_rank=0 min=32484 min_rank=1 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33001, 32848, 32807, 32640, 32768, 32801, 32749, 32530] avg=32768.0000 max=33001 max_rank=0 min=32530 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33152, 32484, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33152 max_rank=0 min=32484 min_rank=1 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33096 max_rank=0 min=32515 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33096 max_rank=0 min=32515 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33096 max_rank=0 min=32515 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33152, 32484, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33152 max_rank=0 min=32484 min_rank=1 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32540, 32623, 32847, 32654, 32957, 32912, 32515] avg=32768.0000 max=33096 max_rank=0 min=32515 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32763, 32797, 32435, 32877, 32823, 32685, 33013, 32751] avg=32768.0000 max=33013 max_rank=6 min=32435 min_rank=2 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[33112, 32955, 32668, 32613, 33216, 32273, 32684, 32623] avg=32768.0000 max=33216 max_rank=4 min=32273 min_rank=5 imbalance_ratio=1.013672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[33112, 32955, 32668, 32613, 33216, 32273, 32684, 32623] avg=32768.0000 max=33216 max_rank=4 min=32273 min_rank=5 imbalance_ratio=1.013672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] avg=32768.0000 max=33096 max_rank=0 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=120 max_load_reduction_pct=0.003613 moved_to_replicas=136 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[32707, 32833, 32680, 32894, 33023, 32740, 32531, 32736] avg=32768.0000 max=33023 max_rank=4 min=32531 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[33112, 32955, 32668, 32613, 33216, 32273, 32684, 32623] avg=32768.0000 max=33216 max_rank=4 min=32273 min_rank=5 imbalance_ratio=1.013672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] avg=32768.0000 max=33096 max_rank=0 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=120 max_load_reduction_pct=0.003613 moved_to_replicas=136 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] avg=32768.0000 max=33096 max_rank=0 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=120 max_load_reduction_pct=0.003613 moved_to_replicas=136 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[33112, 32955, 32668, 32613, 33216, 32273, 32684, 32623] avg=32768.0000 max=33216 max_rank=4 min=32273 min_rank=5 imbalance_ratio=1.013672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] avg=32768.0000 max=33096 max_rank=0 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=120 max_load_reduction_pct=0.003613 moved_to_replicas=136 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32879, 32865, 32789, 32737, 32676, 32605, 32793, 32800] avg=32768.0000 max=32879 max_rank=0 min=32605 min_rank=5 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[33112, 32955, 32668, 32613, 33216, 32273, 32684, 32623] avg=32768.0000 max=33216 max_rank=4 min=32273 min_rank=5 imbalance_ratio=1.013672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[33112, 32955, 32668, 32613, 33216, 32273, 32684, 32623] avg=32768.0000 max=33216 max_rank=4 min=32273 min_rank=5 imbalance_ratio=1.013672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32922, 32764, 32748, 32761, 32731, 32715, 32776, 32727] avg=32768.0000 max=32922 max_rank=0 min=32715 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[33112, 32955, 32668, 32613, 33216, 32273, 32684, 32623] avg=32768.0000 max=33216 max_rank=4 min=32273 min_rank=5 imbalance_ratio=1.013672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] avg=32768.0000 max=33096 max_rank=0 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=120 max_load_reduction_pct=0.003613 moved_to_replicas=136 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] avg=32768.0000 max=33096 max_rank=0 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=120 max_load_reduction_pct=0.003613 moved_to_replicas=136 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] avg=32768.0000 max=33096 max_rank=0 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=120 max_load_reduction_pct=0.003613 moved_to_replicas=136 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33128, 32666, 32626, 32856, 32485] avg=32768.0000 max=33128 max_rank=3 min=32485 min_rank=7 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[33112, 32955, 32668, 32613, 33216, 32273, 32684, 32623] avg=32768.0000 max=33216 max_rank=4 min=32273 min_rank=5 imbalance_ratio=1.013672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33128, 32666, 32626, 32856, 32485] avg=32768.0000 max=33128 max_rank=3 min=32485 min_rank=7 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] avg=32768.0000 max=33096 max_rank=3 min=32517 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[33096, 32955, 32668, 32613, 33096, 32409, 32684, 32623] avg=32768.0000 max=33096 max_rank=0 min=32409 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=120 max_load_reduction_pct=0.003613 moved_to_replicas=136 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33128, 32666, 32626, 32856, 32485] avg=32768.0000 max=33128 max_rank=3 min=32485 min_rank=7 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] avg=32768.0000 max=33096 max_rank=3 min=32517 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33128, 32666, 32626, 32856, 32485] avg=32768.0000 max=33128 max_rank=3 min=32485 min_rank=7 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] avg=32768.0000 max=33096 max_rank=3 min=32517 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] avg=32768.0000 max=33096 max_rank=3 min=32517 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32808, 32826, 32754, 32808, 32763, 32710, 32721, 32754] avg=32768.0000 max=32826 max_rank=1 min=32710 min_rank=5 imbalance_ratio=1.001770 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33128, 32666, 32626, 32856, 32485] avg=32768.0000 max=33128 max_rank=3 min=32485 min_rank=7 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33128, 32666, 32626, 32856, 32485] avg=32768.0000 max=33128 max_rank=3 min=32485 min_rank=7 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] avg=32768.0000 max=33096 max_rank=3 min=32517 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[32532, 32677, 32778, 32639, 33019, 32767, 32925, 32807] avg=32768.0000 max=33019 max_rank=4 min=32532 min_rank=0 imbalance_ratio=1.007660 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33128, 32666, 32626, 32856, 32485] avg=32768.0000 max=33128 max_rank=3 min=32485 min_rank=7 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] avg=32768.0000 max=33096 max_rank=3 min=32517 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] avg=32768.0000 max=33096 max_rank=3 min=32517 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33128, 32666, 32626, 32856, 32485] avg=32768.0000 max=33128 max_rank=3 min=32485 min_rank=7 imbalance_ratio=1.010986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[32894, 32874, 32615, 33096, 32666, 32626, 32856, 32517] avg=32768.0000 max=33096 max_rank=3 min=32517 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000966 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[33067, 32771, 32778, 32580, 32711, 32630, 32711, 32896] avg=32768.0000 max=33067 max_rank=0 min=32580 min_rank=3 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[32911, 32523, 32855, 32813, 32659, 32623, 33066, 32694] avg=32768.0000 max=33066 max_rank=6 min=32523 min_rank=1 imbalance_ratio=1.009094 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[33021, 32656, 32718, 32928, 32653, 32651, 32808, 32709] avg=32768.0000 max=33021 max_rank=0 min=32651 min_rank=5 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[32781, 32637, 32645, 33003, 32565, 33085, 32646, 32782] avg=32768.0000 max=33085 max_rank=5 min=32565 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[33041, 32878, 32753, 32846, 32650, 32620, 32722, 32634] avg=32768.0000 max=33041 max_rank=0 min=32620 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[32916, 32358, 32561, 32828, 32896, 33040, 32663, 32882] avg=32768.0000 max=33040 max_rank=5 min=32358 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32566, 32739, 32848, 32683, 33162, 32576, 32991] avg=32768.0000 max=33162 max_rank=5 min=32566 min_rank=1 imbalance_ratio=1.012024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32566, 32739, 32848, 32683, 33162, 32576, 32991] avg=32768.0000 max=33162 max_rank=5 min=32566 min_rank=1 imbalance_ratio=1.012024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32566, 32739, 32848, 32683, 33162, 32576, 32991] avg=32768.0000 max=33162 max_rank=5 min=32566 min_rank=1 imbalance_ratio=1.012024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] avg=32768.0000 max=33096 max_rank=5 min=32576 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66 max_load_reduction_pct=0.001990 moved_to_replicas=66 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32566, 32739, 32848, 32683, 33162, 32576, 32991] avg=32768.0000 max=33162 max_rank=5 min=32566 min_rank=1 imbalance_ratio=1.012024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] avg=32768.0000 max=33096 max_rank=5 min=32576 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66 max_load_reduction_pct=0.001990 moved_to_replicas=66 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] avg=32768.0000 max=33096 max_rank=5 min=32576 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66 max_load_reduction_pct=0.001990 moved_to_replicas=66 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[32463, 32896, 32904, 32888, 32736, 32840, 32848, 32569] avg=32768.0000 max=32904 max_rank=2 min=32463 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] avg=32768.0000 max=33096 max_rank=5 min=32576 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66 max_load_reduction_pct=0.001990 moved_to_replicas=66 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=25 tokens_per_ep_rank=[32801, 32422, 32880, 32806, 32838, 32962, 32779, 32656] avg=32768.0000 max=32962 max_rank=5 min=32422 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32566, 32739, 32848, 32683, 33162, 32576, 32991] avg=32768.0000 max=33162 max_rank=5 min=32566 min_rank=1 imbalance_ratio=1.012024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32566, 32739, 32848, 32683, 33162, 32576, 32991] avg=32768.0000 max=33162 max_rank=5 min=32566 min_rank=1 imbalance_ratio=1.012024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] avg=32768.0000 max=33096 max_rank=5 min=32576 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66 max_load_reduction_pct=0.001990 moved_to_replicas=66 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32566, 32739, 32848, 32683, 33162, 32576, 32991] avg=32768.0000 max=33162 max_rank=5 min=32566 min_rank=1 imbalance_ratio=1.012024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] avg=32768.0000 max=33096 max_rank=5 min=32576 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66 max_load_reduction_pct=0.001990 moved_to_replicas=66 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=26 tokens_per_ep_rank=[32768, 32537, 32701, 32841, 32941, 32723, 32671, 32962] avg=32768.0000 max=32962 max_rank=7 min=32537 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] avg=32768.0000 max=33096 max_rank=5 min=32576 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66 max_load_reduction_pct=0.001990 moved_to_replicas=66 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32566, 32739, 32848, 32683, 33162, 32576, 32991] avg=32768.0000 max=33162 max_rank=5 min=32566 min_rank=1 imbalance_ratio=1.012024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=27 tokens_per_ep_rank=[32579, 32632, 32739, 32848, 32683, 33096, 32576, 32991] avg=32768.0000 max=33096 max_rank=5 min=32576 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=66 max_load_reduction_pct=0.001990 moved_to_replicas=66 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32379, 33132, 32962, 32687, 32711, 32838] avg=32768.0000 max=33132 max_rank=3 min=32379 min_rank=2 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=28 tokens_per_ep_rank=[32952, 32791, 32764, 32820, 32698, 32436, 32888, 32795] avg=32768.0000 max=32952 max_rank=0 min=32436 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32379, 33132, 32962, 32687, 32711, 32838] avg=32768.0000 max=33132 max_rank=3 min=32379 min_rank=2 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32379, 33132, 32962, 32687, 32711, 32838] avg=32768.0000 max=33132 max_rank=3 min=32379 min_rank=2 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] avg=32768.0000 max=33096 max_rank=3 min=32415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32379, 33132, 32962, 32687, 32711, 32838] avg=32768.0000 max=33132 max_rank=3 min=32379 min_rank=2 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] avg=32768.0000 max=33096 max_rank=3 min=32415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] avg=32768.0000 max=33096 max_rank=3 min=32415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] avg=32768.0000 max=33096 max_rank=3 min=32415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=29 tokens_per_ep_rank=[32961, 32558, 32664, 33086, 32627, 32620, 33016, 32612] avg=32768.0000 max=33086 max_rank=3 min=32558 min_rank=1 imbalance_ratio=1.009705 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=30 tokens_per_ep_rank=[33003, 32660, 32651, 32864, 32752, 32771, 32572, 32871] avg=32768.0000 max=33003 max_rank=0 min=32572 min_rank=6 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32379, 33132, 32962, 32687, 32711, 32838] avg=32768.0000 max=33132 max_rank=3 min=32379 min_rank=2 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32379, 33132, 32962, 32687, 32711, 32838] avg=32768.0000 max=33132 max_rank=3 min=32379 min_rank=2 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] avg=32768.0000 max=33096 max_rank=3 min=32415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32379, 33132, 32962, 32687, 32711, 32838] avg=32768.0000 max=33132 max_rank=3 min=32379 min_rank=2 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] avg=32768.0000 max=33096 max_rank=3 min=32415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=31 tokens_per_ep_rank=[32553, 33062, 32733, 32598, 32822, 32640, 32985, 32751] avg=32768.0000 max=33062 max_rank=1 min=32553 min_rank=0 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] avg=32768.0000 max=33096 max_rank=3 min=32415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32379, 33132, 32962, 32687, 32711, 32838] avg=32768.0000 max=33132 max_rank=3 min=32379 min_rank=2 imbalance_ratio=1.011108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=32 tokens_per_ep_rank=[32752, 32683, 32415, 33096, 32962, 32687, 32711, 32838] avg=32768.0000 max=33096 max_rank=3 min=32415 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36 max_load_reduction_pct=0.001087 moved_to_replicas=36 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32571, 33302, 32979, 32578] avg=32768.0000 max=33302 max_rank=5 min=32571 min_rank=4 imbalance_ratio=1.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32571, 33302, 32979, 32578] avg=32768.0000 max=33302 max_rank=5 min=32571 min_rank=4 imbalance_ratio=1.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=33 tokens_per_ep_rank=[32456, 32608, 32564, 32983, 33087, 32756, 32882, 32808] avg=32768.0000 max=33087 max_rank=4 min=32456 min_rank=0 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32571, 33302, 32979, 32578] avg=32768.0000 max=33302 max_rank=5 min=32571 min_rank=4 imbalance_ratio=1.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] avg=32768.0000 max=33096 max_rank=5 min=32573 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=206 max_load_reduction_pct=0.006186 moved_to_replicas=206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32571, 33302, 32979, 32578] avg=32768.0000 max=33302 max_rank=5 min=32571 min_rank=4 imbalance_ratio=1.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] avg=32768.0000 max=33096 max_rank=5 min=32573 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=206 max_load_reduction_pct=0.006186 moved_to_replicas=206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] avg=32768.0000 max=33096 max_rank=5 min=32573 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=206 max_load_reduction_pct=0.006186 moved_to_replicas=206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] avg=32768.0000 max=33096 max_rank=5 min=32573 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=206 max_load_reduction_pct=0.006186 moved_to_replicas=206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=34 tokens_per_ep_rank=[32592, 32931, 32878, 32745, 32806, 32772, 32617, 32803] avg=32768.0000 max=32931 max_rank=1 min=32592 min_rank=0 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33097, 32634, 32589, 32888, 32480, 32697, 32962] avg=32768.0000 max=33097 max_rank=1 min=32480 min_rank=5 imbalance_ratio=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33097, 32634, 32589, 32888, 32480, 32697, 32962] avg=32768.0000 max=33097 max_rank=1 min=32480 min_rank=5 imbalance_ratio=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33097, 32634, 32589, 32888, 32480, 32697, 32962] avg=32768.0000 max=33097 max_rank=1 min=32480 min_rank=5 imbalance_ratio=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=35 tokens_per_ep_rank=[32836, 32806, 32520, 32971, 32679, 32913, 32638, 32781] avg=32768.0000 max=32971 max_rank=3 min=32520 min_rank=2 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32571, 33302, 32979, 32578] avg=32768.0000 max=33302 max_rank=5 min=32571 min_rank=4 imbalance_ratio=1.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33097, 32634, 32589, 32888, 32480, 32697, 32962] avg=32768.0000 max=33097 max_rank=1 min=32480 min_rank=5 imbalance_ratio=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000030 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000030 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000030 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32571, 33302, 32979, 32578] avg=32768.0000 max=33302 max_rank=5 min=32571 min_rank=4 imbalance_ratio=1.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] avg=32768.0000 max=33096 max_rank=5 min=32573 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=206 max_load_reduction_pct=0.006186 moved_to_replicas=206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000030 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] avg=32768.0000 max=33096 max_rank=5 min=32573 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=206 max_load_reduction_pct=0.006186 moved_to_replicas=206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32571, 33302, 32979, 32578] avg=32768.0000 max=33302 max_rank=5 min=32571 min_rank=4 imbalance_ratio=1.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=36 tokens_per_ep_rank=[32797, 32743, 32807, 32711, 32808, 32651, 32772, 32855] avg=32768.0000 max=32855 max_rank=7 min=32651 min_rank=5 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] avg=32768.0000 max=33096 max_rank=5 min=32573 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=206 max_load_reduction_pct=0.006186 moved_to_replicas=206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32571, 33302, 32979, 32578] avg=32768.0000 max=33302 max_rank=5 min=32571 min_rank=4 imbalance_ratio=1.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=37 tokens_per_ep_rank=[32573, 32767, 32687, 32687, 32777, 33096, 32979, 32578] avg=32768.0000 max=33096 max_rank=5 min=32573 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=206 max_load_reduction_pct=0.006186 moved_to_replicas=206 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33097, 32634, 32589, 32888, 32480, 32697, 32962] avg=32768.0000 max=33097 max_rank=1 min=32480 min_rank=5 imbalance_ratio=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33097, 32634, 32589, 32888, 32480, 32697, 32962] avg=32768.0000 max=33097 max_rank=1 min=32480 min_rank=5 imbalance_ratio=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000030 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33097, 32634, 32589, 32888, 32480, 32697, 32962] avg=32768.0000 max=33097 max_rank=1 min=32480 min_rank=5 imbalance_ratio=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000030 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=38 tokens_per_ep_rank=[32940, 32776, 32495, 32911, 32848, 32681, 32694, 32799] avg=32768.0000 max=32940 max_rank=0 min=32495 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000030 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33097, 32634, 32589, 32888, 32480, 32697, 32962] avg=32768.0000 max=33097 max_rank=1 min=32480 min_rank=5 imbalance_ratio=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=39 tokens_per_ep_rank=[32797, 33096, 32634, 32589, 32888, 32481, 32697, 32962] avg=32768.0000 max=33096 max_rank=1 min=32481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000030 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=40 tokens_per_ep_rank=[32647, 32736, 32937, 32660, 32810, 32924, 32588, 32842] avg=32768.0000 max=32937 max_rank=2 min=32588 min_rank=6 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=41 tokens_per_ep_rank=[32855, 32640, 32960, 32796, 32580, 32482, 33031, 32800] avg=32768.0000 max=33031 max_rank=6 min=32482 min_rank=5 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32436, 32659, 32606, 32937, 32773, 33158, 32994] avg=32768.0000 max=33158 max_rank=6 min=32436 min_rank=1 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32436, 32659, 32606, 32937, 32773, 33158, 32994] avg=32768.0000 max=33158 max_rank=6 min=32436 min_rank=1 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32436, 32659, 32606, 32937, 32773, 33158, 32994] avg=32768.0000 max=33158 max_rank=6 min=32436 min_rank=1 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32436, 32659, 32606, 32937, 32773, 33158, 32994] avg=32768.0000 max=33158 max_rank=6 min=32436 min_rank=1 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=42 tokens_per_ep_rank=[32697, 32394, 32508, 32871, 32848, 32994, 33077, 32755] avg=32768.0000 max=33077 max_rank=6 min=32394 min_rank=1 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] avg=32768.0000 max=33096 max_rank=6 min=32498 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] avg=32768.0000 max=33096 max_rank=6 min=32498 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] avg=32768.0000 max=33096 max_rank=6 min=32498 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] avg=32768.0000 max=33096 max_rank=6 min=32498 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=43 tokens_per_ep_rank=[32863, 32568, 32678, 32822, 32750, 32793, 32943, 32727] avg=32768.0000 max=32943 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32436, 32659, 32606, 32937, 32773, 33158, 32994] avg=32768.0000 max=33158 max_rank=6 min=32436 min_rank=1 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=44 tokens_per_ep_rank=[32801, 32463, 32850, 32648, 32757, 33005, 32825, 32795] avg=32768.0000 max=33005 max_rank=5 min=32463 min_rank=1 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32436, 32659, 32606, 32937, 32773, 33158, 32994] avg=32768.0000 max=33158 max_rank=6 min=32436 min_rank=1 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] avg=32768.0000 max=33096 max_rank=6 min=32498 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] avg=32768.0000 max=33096 max_rank=6 min=32498 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32436, 32659, 32606, 32937, 32773, 33158, 32994] avg=32768.0000 max=33158 max_rank=6 min=32436 min_rank=1 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=45 tokens_per_ep_rank=[32917, 32747, 32764, 32648, 32873, 32675, 32695, 32825] avg=32768.0000 max=32917 max_rank=0 min=32648 min_rank=3 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] avg=32768.0000 max=33096 max_rank=6 min=32498 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32436, 32659, 32606, 32937, 32773, 33158, 32994] avg=32768.0000 max=33158 max_rank=6 min=32436 min_rank=1 imbalance_ratio=1.011902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=46 tokens_per_ep_rank=[32581, 32498, 32659, 32606, 32937, 32773, 33096, 32994] avg=32768.0000 max=33096 max_rank=6 min=32498 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=62 max_load_reduction_pct=0.001870 moved_to_replicas=62 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=47 tokens_per_ep_rank=[32709, 32461, 32904, 32649, 32945, 33025, 32649, 32802] avg=32768.0000 max=33025 max_rank=5 min=32461 min_rank=1 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=48 tokens_per_ep_rank=[32880, 32714, 33024, 32701, 32559, 32882, 32490, 32894] avg=32768.0000 max=33024 max_rank=2 min=32490 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=48 total_expected_recv=1573108 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344960 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=48 total_expected_recv=1571158 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=347904 peak_demand=33045 required_capacity=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=48 total_expected_recv=1573418 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343936 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=48 total_expected_recv=1572610 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345472 peak_demand=32994 required_capacity=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=48 total_expected_recv=1573662 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342656 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=48 total_expected_recv=1570073 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=347264 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=48 total_expected_recv=1574602 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=340864 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=48 total_expected_recv=1574281 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343936 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:48:21] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 710.0 | throughput per GPU (TFLOP/s/GPU): 125.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.805068E+00 | loss scale: 1.0 | grad norm: 3.250 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32593 expected_overflow=0 exact_recv_per_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32926 expected_overflow=0 exact_recv_per_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32593 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32926 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33041 expected_overflow=0 exact_recv_per_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32366 expected_overflow=0 exact_recv_per_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32926 expected_overflow=0 exact_recv_per_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33041 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32366 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32926 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32483 expected_overflow=0 exact_recv_per_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32855 expected_overflow=0 exact_recv_per_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32855 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32483 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32553 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32799 expected_overflow=0 exact_recv_per_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32799 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32187 expected_overflow=0 exact_recv_per_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32187 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32960 expected_overflow=0 exact_recv_per_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32550 expected_overflow=0 exact_recv_per_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32960 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32550 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33012 expected_overflow=0 exact_recv_per_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33012 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32386 expected_overflow=0 exact_recv_per_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32386 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32561 expected_overflow=0 exact_recv_per_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32561 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32660 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32987 expected_overflow=0 exact_recv_per_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32578 expected_overflow=0 exact_recv_per_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32987 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32578 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32948 expected_overflow=0 exact_recv_per_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32948 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32974 expected_overflow=0 exact_recv_per_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32974 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32910 expected_overflow=0 exact_recv_per_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32910 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32401 expected_overflow=0 exact_recv_per_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32401 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32754 expected_overflow=0 exact_recv_per_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32754 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33012 expected_overflow=0 exact_recv_per_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=40960 expected_recv=33012 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32507 expected_overflow=0 exact_recv_per_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32507 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32995 expected_overflow=0 exact_recv_per_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32995 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32950 expected_overflow=0 exact_recv_per_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32950 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32342 expected_overflow=0 exact_recv_per_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=40960 expected_recv=33039 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32342 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32534 expected_overflow=0 exact_recv_per_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=40960 expected_recv=32534 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32434 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=40960 expected_recv=32837 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32534 expected_overflow=0 exact_recv_per_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32534 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32488 expected_overflow=0 exact_recv_per_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=40960 expected_recv=32488 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32284 expected_overflow=0 exact_recv_per_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32284 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32950 expected_overflow=0 exact_recv_per_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32950 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32937 expected_overflow=0 exact_recv_per_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32937 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32438 expected_overflow=0 exact_recv_per_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32438 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=40960 expected_recv=32460 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=23 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33052 expected_overflow=0 exact_recv_per_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=23 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=24 calls=1 budget=40960 expected_recv=33052 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=24 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=25 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33016 expected_overflow=0 exact_recv_per_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=26 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=26 calls=1 budget=40960 expected_recv=33016 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32486 expected_overflow=0 exact_recv_per_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32486 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=27 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=28 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32593 expected_overflow=0 exact_recv_per_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32593 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33022 expected_overflow=0 exact_recv_per_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32534 expected_overflow=0 exact_recv_per_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=29 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=29 calls=1 budget=40960 expected_recv=33022 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32534 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=30 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32957 expected_overflow=0 exact_recv_per_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32957 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32506 expected_overflow=0 exact_recv_per_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=31 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32506 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32452 expected_overflow=0 exact_recv_per_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=32 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32452 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32945 expected_overflow=0 exact_recv_per_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32320 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32945 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33057 expected_overflow=0 exact_recv_per_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=33 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=34 calls=1 budget=40960 expected_recv=33057 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32510 expected_overflow=0 exact_recv_per_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32510 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=34 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=35 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=37 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=36 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32974 expected_overflow=0 exact_recv_per_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=37 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32974 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=38 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33070 expected_overflow=0 exact_recv_per_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=40 calls=1 budget=40960 expected_recv=33070 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=38 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=41 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=39 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32598 expected_overflow=0 exact_recv_per_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32483 expected_overflow=0 exact_recv_per_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32598 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32483 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32949 expected_overflow=0 exact_recv_per_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=40 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32949 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32511 expected_overflow=0 exact_recv_per_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=41 calls=1 budget=40960 expected_recv=32511 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32536 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33037 expected_overflow=0 exact_recv_per_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=44 calls=1 budget=40960 expected_recv=33037 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=42 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=42 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32536 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32713 expected_overflow=0 exact_recv_per_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32713 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32396 expected_overflow=0 exact_recv_per_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32396 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=43 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32513 expected_overflow=0 exact_recv_per_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32513 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32998 expected_overflow=0 exact_recv_per_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32998 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=44 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=45 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=46 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=47 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32824, 32889, 32917, 32926, 32771, 32632, 32593, 32592] avg=32768.0000 max=32926 max_rank=3 min=32592 min_rank=7 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32166, 32799, 32940, 32682, 33117, 32766, 32847] avg=32768.0000 max=33117 max_rank=5 min=32166 min_rank=1 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33041, 32763, 32670, 33016, 32926, 32366, 32713, 32649] avg=32768.0000 max=33041 max_rank=0 min=32366 min_rank=5 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32166, 32799, 32940, 32682, 33117, 32766, 32847] avg=32768.0000 max=33117 max_rank=5 min=32166 min_rank=1 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] avg=32768.0000 max=33096 max_rank=5 min=32187 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32166, 32799, 32940, 32682, 33117, 32766, 32847] avg=32768.0000 max=33117 max_rank=5 min=32166 min_rank=1 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32166, 32799, 32940, 32682, 33117, 32766, 32847] avg=32768.0000 max=33117 max_rank=5 min=32166 min_rank=1 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] avg=32768.0000 max=33096 max_rank=5 min=32187 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] avg=32768.0000 max=33096 max_rank=5 min=32187 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] avg=32768.0000 max=33096 max_rank=5 min=32187 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32166, 32799, 32940, 32682, 33117, 32766, 32847] avg=32768.0000 max=33117 max_rank=5 min=32166 min_rank=1 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32714, 32802, 32815, 32855, 32483, 32826, 33096, 32553] avg=32768.0000 max=33096 max_rank=6 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] avg=32768.0000 max=33096 max_rank=5 min=32187 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32166, 32799, 32940, 32682, 33117, 32766, 32847] avg=32768.0000 max=33117 max_rank=5 min=32166 min_rank=1 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32166, 32799, 32940, 32682, 33117, 32766, 32847] avg=32768.0000 max=33117 max_rank=5 min=32166 min_rank=1 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32508, 32556, 32751, 32869, 32797, 32986, 32649, 33028] avg=32768.0000 max=33028 max_rank=7 min=32508 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] avg=32768.0000 max=33096 max_rank=5 min=32187 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] avg=32768.0000 max=33096 max_rank=5 min=32187 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33169, 32663, 32650, 32781] avg=32768.0000 max=33169 max_rank=4 min=32650 min_rank=6 imbalance_ratio=1.012238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32166, 32799, 32940, 32682, 33117, 32766, 32847] avg=32768.0000 max=33117 max_rank=5 min=32166 min_rank=1 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33169, 32663, 32650, 32781] avg=32768.0000 max=33169 max_rank=4 min=32650 min_rank=6 imbalance_ratio=1.012238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] avg=32768.0000 max=33096 max_rank=4 min=32663 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=73 max_load_reduction_pct=0.002201 moved_to_replicas=73 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32827, 32187, 32799, 32940, 32682, 33096, 32766, 32847] avg=32768.0000 max=33096 max_rank=5 min=32187 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33169, 32663, 32650, 32781] avg=32768.0000 max=33169 max_rank=4 min=32650 min_rank=6 imbalance_ratio=1.012238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33169, 32663, 32650, 32781] avg=32768.0000 max=33169 max_rank=4 min=32650 min_rank=6 imbalance_ratio=1.012238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] avg=32768.0000 max=33096 max_rank=4 min=32663 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=73 max_load_reduction_pct=0.002201 moved_to_replicas=73 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] avg=32768.0000 max=33096 max_rank=4 min=32663 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=73 max_load_reduction_pct=0.002201 moved_to_replicas=73 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] avg=32768.0000 max=33096 max_rank=4 min=32663 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=73 max_load_reduction_pct=0.002201 moved_to_replicas=73 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33169, 32663, 32650, 32781] avg=32768.0000 max=33169 max_rank=4 min=32650 min_rank=6 imbalance_ratio=1.012238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] avg=32768.0000 max=33096 max_rank=4 min=32663 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=73 max_load_reduction_pct=0.002201 moved_to_replicas=73 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[32902, 32530, 32588, 32550, 32960, 32779, 32891, 32944] avg=32768.0000 max=32960 max_rank=4 min=32530 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33169, 32663, 32650, 32781] avg=32768.0000 max=33169 max_rank=4 min=32650 min_rank=6 imbalance_ratio=1.012238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33169, 32663, 32650, 32781] avg=32768.0000 max=33169 max_rank=4 min=32650 min_rank=6 imbalance_ratio=1.012238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[33063, 32608, 33012, 32763, 32888, 32933, 32491, 32386] avg=32768.0000 max=33063 max_rank=0 min=32386 min_rank=7 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] avg=32768.0000 max=33096 max_rank=4 min=32663 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=73 max_load_reduction_pct=0.002201 moved_to_replicas=73 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] avg=32768.0000 max=33096 max_rank=4 min=32663 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=73 max_load_reduction_pct=0.002201 moved_to_replicas=73 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33169, 32663, 32650, 32781] avg=32768.0000 max=33169 max_rank=4 min=32650 min_rank=6 imbalance_ratio=1.012238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[32670, 32761, 32759, 32691, 33096, 32663, 32723, 32781] avg=32768.0000 max=33096 max_rank=4 min=32663 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=73 max_load_reduction_pct=0.002201 moved_to_replicas=73 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32859, 32729, 32886, 32715, 32832, 32666, 32844, 32613] avg=32768.0000 max=32886 max_rank=2 min=32613 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32760, 32668, 32907, 32987, 32660, 32561, 32816, 32785] avg=32768.0000 max=32987 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.006683 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[32786, 32771, 32578, 32887, 32750, 32948, 32555, 32869] avg=32768.0000 max=32948 max_rank=5 min=32555 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32910, 32532, 32711, 32944, 32603, 32734, 32974, 32736] avg=32768.0000 max=32974 max_rank=6 min=32532 min_rank=1 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32851, 32946, 32401, 32737, 32756, 32795, 32765, 32893] avg=32768.0000 max=32946 max_rank=1 min=32401 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32864, 32680, 32723, 32690, 32791, 32968, 32706, 32722] avg=32768.0000 max=32968 max_rank=5 min=32680 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[33001, 32754, 32851, 32688, 32652, 32715, 32692, 32791] avg=32768.0000 max=33001 max_rank=0 min=32652 min_rank=4 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[32950, 32995, 33012, 32792, 32622, 32714, 32507, 32552] avg=32768.0000 max=33012 max_rank=2 min=32507 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[32429, 32675, 33338, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33338 max_rank=2 min=32429 min_rank=0 imbalance_ratio=1.017395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[32984, 33039, 32766, 32534, 32342, 32996, 32751, 32732] avg=32768.0000 max=33039 max_rank=1 min=32342 min_rank=4 imbalance_ratio=1.008270 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[32429, 32675, 33338, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33338 max_rank=2 min=32429 min_rank=0 imbalance_ratio=1.017395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33096 max_rank=2 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=242 max_load_reduction_pct=0.007259 moved_to_replicas=242 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[32429, 32675, 33338, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33338 max_rank=2 min=32429 min_rank=0 imbalance_ratio=1.017395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[32429, 32675, 33338, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33338 max_rank=2 min=32429 min_rank=0 imbalance_ratio=1.017395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33096 max_rank=2 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=242 max_load_reduction_pct=0.007259 moved_to_replicas=242 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33096 max_rank=2 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=242 max_load_reduction_pct=0.007259 moved_to_replicas=242 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33096 max_rank=2 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=242 max_load_reduction_pct=0.007259 moved_to_replicas=242 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[32429, 32675, 33338, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33338 max_rank=2 min=32429 min_rank=0 imbalance_ratio=1.017395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[32980, 32888, 32549, 32837, 32434, 32729, 32869, 32858] avg=32768.0000 max=32980 max_rank=0 min=32434 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33096 max_rank=2 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=242 max_load_reduction_pct=0.007259 moved_to_replicas=242 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32424, 32950, 33110, 32710, 32937, 32768, 32785] avg=32768.0000 max=33110 max_rank=3 min=32424 min_rank=1 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32424, 32950, 33110, 32710, 32937, 32768, 32785] avg=32768.0000 max=33110 max_rank=3 min=32424 min_rank=1 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[32429, 32675, 33338, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33338 max_rank=2 min=32429 min_rank=0 imbalance_ratio=1.017395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[32930, 32816, 32734, 32488, 32835, 32841, 32966, 32534] avg=32768.0000 max=32966 max_rank=6 min=32488 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] avg=32768.0000 max=33096 max_rank=3 min=32438 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32424, 32950, 33110, 32710, 32937, 32768, 32785] avg=32768.0000 max=33110 max_rank=3 min=32424 min_rank=1 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32424, 32950, 33110, 32710, 32937, 32768, 32785] avg=32768.0000 max=33110 max_rank=3 min=32424 min_rank=1 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] avg=32768.0000 max=33096 max_rank=3 min=32438 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33096 max_rank=2 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=242 max_load_reduction_pct=0.007259 moved_to_replicas=242 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] avg=32768.0000 max=33096 max_rank=3 min=32438 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[32429, 32675, 33338, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33338 max_rank=2 min=32429 min_rank=0 imbalance_ratio=1.017395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] avg=32768.0000 max=33096 max_rank=3 min=32438 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33143, 32792, 32644, 32539] avg=32768.0000 max=33143 max_rank=4 min=32539 min_rank=7 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32424, 32950, 33110, 32710, 32937, 32768, 32785] avg=32768.0000 max=33110 max_rank=3 min=32424 min_rank=1 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33143, 32792, 32644, 32539] avg=32768.0000 max=33143 max_rank=4 min=32539 min_rank=7 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33096 max_rank=2 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=242 max_load_reduction_pct=0.007259 moved_to_replicas=242 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] avg=32768.0000 max=33096 max_rank=4 min=32576 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33143, 32792, 32644, 32539] avg=32768.0000 max=33143 max_rank=4 min=32539 min_rank=7 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] avg=32768.0000 max=33096 max_rank=3 min=32438 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[32774, 32810, 32696, 32621, 32801, 32885, 32733, 32824] avg=32768.0000 max=32885 max_rank=5 min=32621 min_rank=3 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33143, 32792, 32644, 32539] avg=32768.0000 max=33143 max_rank=4 min=32539 min_rank=7 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] avg=32768.0000 max=33096 max_rank=4 min=32576 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] avg=32768.0000 max=33096 max_rank=4 min=32576 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] avg=32768.0000 max=33096 max_rank=4 min=32576 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33143, 32792, 32644, 32539] avg=32768.0000 max=33143 max_rank=4 min=32539 min_rank=7 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[32429, 32675, 33338, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33338 max_rank=2 min=32429 min_rank=0 imbalance_ratio=1.017395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32424, 32950, 33110, 32710, 32937, 32768, 32785] avg=32768.0000 max=33110 max_rank=3 min=32424 min_rank=1 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] avg=32768.0000 max=33096 max_rank=4 min=32576 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[32671, 32675, 33096, 32929, 32647, 32812, 32504, 32810] avg=32768.0000 max=33096 max_rank=2 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=242 max_load_reduction_pct=0.007259 moved_to_replicas=242 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] avg=32768.0000 max=33096 max_rank=3 min=32438 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32424, 32950, 33110, 32710, 32937, 32768, 32785] avg=32768.0000 max=33110 max_rank=3 min=32424 min_rank=1 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] avg=32768.0000 max=33096 max_rank=3 min=32438 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33143, 32792, 32644, 32539] avg=32768.0000 max=33143 max_rank=4 min=32539 min_rank=7 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[32819, 32712, 32952, 32789, 32841, 32880, 32867, 32284] avg=32768.0000 max=32952 max_rank=2 min=32284 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] avg=32768.0000 max=33096 max_rank=4 min=32576 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33143, 32792, 32644, 32539] avg=32768.0000 max=33143 max_rank=4 min=32539 min_rank=7 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33209, 32432, 32812] avg=32768.0000 max=33209 max_rank=5 min=32432 min_rank=6 imbalance_ratio=1.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33209, 32432, 32812] avg=32768.0000 max=33209 max_rank=5 min=32432 min_rank=6 imbalance_ratio=1.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32424, 32950, 33110, 32710, 32937, 32768, 32785] avg=32768.0000 max=33110 max_rank=3 min=32424 min_rank=1 imbalance_ratio=1.010437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] avg=32768.0000 max=33096 max_rank=4 min=32576 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33209, 32432, 32812] avg=32768.0000 max=33209 max_rank=5 min=32432 min_rank=6 imbalance_ratio=1.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] avg=32768.0000 max=33096 max_rank=5 min=32467 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=113 max_load_reduction_pct=0.003403 moved_to_replicas=113 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33209, 32432, 32812] avg=32768.0000 max=33209 max_rank=5 min=32432 min_rank=6 imbalance_ratio=1.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] avg=32768.0000 max=33096 max_rank=5 min=32467 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=113 max_load_reduction_pct=0.003403 moved_to_replicas=113 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[32460, 32438, 32950, 33096, 32710, 32937, 32768, 32785] avg=32768.0000 max=33096 max_rank=3 min=32438 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=14 max_load_reduction_pct=0.000423 moved_to_replicas=14 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] avg=32768.0000 max=33096 max_rank=5 min=32467 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=113 max_load_reduction_pct=0.003403 moved_to_replicas=113 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] avg=32768.0000 max=33096 max_rank=5 min=32467 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=113 max_load_reduction_pct=0.003403 moved_to_replicas=113 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33209, 32432, 32812] avg=32768.0000 max=33209 max_rank=5 min=32432 min_rank=6 imbalance_ratio=1.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33143, 32792, 32644, 32539] avg=32768.0000 max=33143 max_rank=4 min=32539 min_rank=7 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] avg=32768.0000 max=33096 max_rank=5 min=32467 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=113 max_load_reduction_pct=0.003403 moved_to_replicas=113 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[32580, 32996, 32576, 32874, 33096, 32792, 32644, 32586] avg=32768.0000 max=33096 max_rank=4 min=32576 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33209, 32432, 32812] avg=32768.0000 max=33209 max_rank=5 min=32432 min_rank=6 imbalance_ratio=1.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=25 tokens_per_ep_rank=[32694, 32740, 32690, 32911, 32793, 32613, 32651, 33052] avg=32768.0000 max=33052 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] avg=32768.0000 max=33096 max_rank=5 min=32467 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=113 max_load_reduction_pct=0.003403 moved_to_replicas=113 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33209, 32432, 32812] avg=32768.0000 max=33209 max_rank=5 min=32432 min_rank=6 imbalance_ratio=1.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] avg=32768.0000 max=33096 max_rank=5 min=32467 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=113 max_load_reduction_pct=0.003403 moved_to_replicas=113 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=26 tokens_per_ep_rank=[32665, 32851, 32805, 32917, 32698, 32763, 32709, 32736] avg=32768.0000 max=32917 max_rank=3 min=32665 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33209, 32432, 32812] avg=32768.0000 max=33209 max_rank=5 min=32432 min_rank=6 imbalance_ratio=1.013458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=27 tokens_per_ep_rank=[32537, 32467, 32686, 33016, 32985, 33096, 32545, 32812] avg=32768.0000 max=33096 max_rank=5 min=32467 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=113 max_load_reduction_pct=0.003403 moved_to_replicas=113 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=28 tokens_per_ep_rank=[32631, 32858, 32724, 32876, 32585, 32699, 32786, 32985] avg=32768.0000 max=32985 max_rank=7 min=32585 min_rank=4 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=29 tokens_per_ep_rank=[32690, 32738, 32929, 32880, 32939, 32486, 32679, 32803] avg=32768.0000 max=32939 max_rank=4 min=32486 min_rank=5 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=30 tokens_per_ep_rank=[32736, 32752, 32738, 33022, 32502, 32859, 32887, 32648] avg=32768.0000 max=33022 max_rank=3 min=32502 min_rank=4 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=31 tokens_per_ep_rank=[32655, 32534, 32971, 32840, 32695, 32593, 32905, 32951] avg=32768.0000 max=32971 max_rank=2 min=32534 min_rank=1 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=32 tokens_per_ep_rank=[32963, 32683, 32952, 32881, 32566, 32768, 32608, 32723] avg=32768.0000 max=32963 max_rank=0 min=32566 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32581, 32678, 32662, 33179, 32725, 32802] avg=32768.0000 max=33179 max_rank=5 min=32581 min_rank=2 imbalance_ratio=1.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=33 tokens_per_ep_rank=[32857, 32506, 32827, 32940, 32645, 32957, 32823, 32589] avg=32768.0000 max=32957 max_rank=5 min=32506 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32581, 32678, 32662, 33179, 32725, 32802] avg=32768.0000 max=33179 max_rank=5 min=32581 min_rank=2 imbalance_ratio=1.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32581, 32678, 32662, 33179, 32725, 32802] avg=32768.0000 max=33179 max_rank=5 min=32581 min_rank=2 imbalance_ratio=1.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] avg=32768.0000 max=33096 max_rank=5 min=32662 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.002502 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32581, 32678, 32662, 33179, 32725, 32802] avg=32768.0000 max=33179 max_rank=5 min=32581 min_rank=2 imbalance_ratio=1.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] avg=32768.0000 max=33096 max_rank=5 min=32662 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.002502 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] avg=32768.0000 max=33096 max_rank=5 min=32662 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.002502 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] avg=32768.0000 max=33096 max_rank=5 min=32662 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.002502 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32581, 32678, 32662, 33179, 32725, 32802] avg=32768.0000 max=33179 max_rank=5 min=32581 min_rank=2 imbalance_ratio=1.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=39 tokens_per_ep_rank=[33116, 32721, 32695, 32883, 32744, 32722, 32820, 32443] avg=32768.0000 max=33116 max_rank=0 min=32443 min_rank=7 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=34 tokens_per_ep_rank=[32818, 32452, 32761, 32879, 32976, 32880, 32733, 32645] avg=32768.0000 max=32976 max_rank=4 min=32452 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=39 tokens_per_ep_rank=[33116, 32721, 32695, 32883, 32744, 32722, 32820, 32443] avg=32768.0000 max=33116 max_rank=0 min=32443 min_rank=7 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] avg=32768.0000 max=33096 max_rank=5 min=32662 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.002502 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=39 tokens_per_ep_rank=[33116, 32721, 32695, 32883, 32744, 32722, 32820, 32443] avg=32768.0000 max=33116 max_rank=0 min=32443 min_rank=7 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=39 tokens_per_ep_rank=[33116, 32721, 32695, 32883, 32744, 32722, 32820, 32443] avg=32768.0000 max=33116 max_rank=0 min=32443 min_rank=7 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=39 tokens_per_ep_rank=[33116, 32721, 32695, 32883, 32744, 32722, 32820, 32443] avg=32768.0000 max=33116 max_rank=0 min=32443 min_rank=7 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=35 tokens_per_ep_rank=[32510, 33057, 32921, 32576, 32859, 32956, 32320, 32945] avg=32768.0000 max=33057 max_rank=1 min=32320 min_rank=6 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32581, 32678, 32662, 33179, 32725, 32802] avg=32768.0000 max=33179 max_rank=5 min=32581 min_rank=2 imbalance_ratio=1.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=36 tokens_per_ep_rank=[32912, 32545, 32840, 32728, 32992, 32644, 32603, 32880] avg=32768.0000 max=32992 max_rank=4 min=32545 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] avg=32768.0000 max=33096 max_rank=5 min=32662 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.002502 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32581, 32678, 32662, 33179, 32725, 32802] avg=32768.0000 max=33179 max_rank=5 min=32581 min_rank=2 imbalance_ratio=1.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32433, 32483, 32681, 32986, 33174] avg=32768.0000 max=33174 max_rank=7 min=32433 min_rank=3 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=39 tokens_per_ep_rank=[33116, 32721, 32695, 32883, 32744, 32722, 32820, 32443] avg=32768.0000 max=33116 max_rank=0 min=32443 min_rank=7 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] avg=32768.0000 max=33096 max_rank=5 min=32662 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.002502 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32433, 32483, 32681, 32986, 33174] avg=32768.0000 max=33174 max_rank=7 min=32433 min_rank=3 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32433, 32483, 32681, 32986, 33174] avg=32768.0000 max=33174 max_rank=7 min=32433 min_rank=3 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=37 tokens_per_ep_rank=[32684, 32976, 32668, 32889, 32551, 32840, 32887, 32649] avg=32768.0000 max=32976 max_rank=1 min=32551 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32433, 32483, 32681, 32986, 33174] avg=32768.0000 max=33174 max_rank=7 min=32433 min_rank=3 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] avg=32768.0000 max=33096 max_rank=7 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] avg=32768.0000 max=33096 max_rank=7 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] avg=32768.0000 max=33096 max_rank=7 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=39 tokens_per_ep_rank=[33116, 32721, 32695, 32883, 32744, 32722, 32820, 32443] avg=32768.0000 max=33116 max_rank=0 min=32443 min_rank=7 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] avg=32768.0000 max=33096 max_rank=7 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32581, 32678, 32662, 33179, 32725, 32802] avg=32768.0000 max=33179 max_rank=5 min=32581 min_rank=2 imbalance_ratio=1.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32433, 32483, 32681, 32986, 33174] avg=32768.0000 max=33174 max_rank=7 min=32433 min_rank=3 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=43 tokens_per_ep_rank=[33188, 32828, 32508, 32733, 32738, 32949, 32827, 32373] avg=32768.0000 max=33188 max_rank=0 min=32373 min_rank=7 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=43 tokens_per_ep_rank=[33188, 32828, 32508, 32733, 32738, 32949, 32827, 32373] avg=32768.0000 max=33188 max_rank=0 min=32373 min_rank=7 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=43 tokens_per_ep_rank=[33188, 32828, 32508, 32733, 32738, 32949, 32827, 32373] avg=32768.0000 max=33188 max_rank=0 min=32373 min_rank=7 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=38 tokens_per_ep_rank=[32824, 32693, 32664, 32678, 32662, 33096, 32725, 32802] avg=32768.0000 max=33096 max_rank=5 min=32662 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.002502 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] avg=32768.0000 max=33096 max_rank=7 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=43 tokens_per_ep_rank=[33188, 32828, 32508, 32733, 32738, 32949, 32827, 32373] avg=32768.0000 max=33188 max_rank=0 min=32373 min_rank=7 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=39 tokens_per_ep_rank=[33116, 32721, 32695, 32883, 32744, 32722, 32820, 32443] avg=32768.0000 max=33116 max_rank=0 min=32443 min_rank=7 imbalance_ratio=1.010620 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=43 tokens_per_ep_rank=[33188, 32828, 32508, 32733, 32738, 32949, 32827, 32373] avg=32768.0000 max=33188 max_rank=0 min=32373 min_rank=7 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=39 tokens_per_ep_rank=[33096, 32721, 32695, 32883, 32744, 32722, 32820, 32463] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=20 max_load_reduction_pct=0.000604 moved_to_replicas=20 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32433, 32483, 32681, 32986, 33174] avg=32768.0000 max=33174 max_rank=7 min=32433 min_rank=3 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=40 tokens_per_ep_rank=[32457, 32889, 32974, 32743, 32661, 32581, 32940, 32899] avg=32768.0000 max=32974 max_rank=2 min=32457 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] avg=32768.0000 max=33096 max_rank=7 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32433, 32483, 32681, 32986, 33174] avg=32768.0000 max=33174 max_rank=7 min=32433 min_rank=3 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=43 tokens_per_ep_rank=[33188, 32828, 32508, 32733, 32738, 32949, 32827, 32373] avg=32768.0000 max=33188 max_rank=0 min=32373 min_rank=7 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] avg=32768.0000 max=33096 max_rank=7 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=41 tokens_per_ep_rank=[32665, 32598, 32790, 32606, 32703, 32909, 32803, 33070] avg=32768.0000 max=33070 max_rank=7 min=32598 min_rank=1 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=43 tokens_per_ep_rank=[33188, 32828, 32508, 32733, 32738, 32949, 32827, 32373] avg=32768.0000 max=33188 max_rank=0 min=32373 min_rank=7 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32433, 32483, 32681, 32986, 33174] avg=32768.0000 max=33174 max_rank=7 min=32433 min_rank=3 imbalance_ratio=1.012390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=42 tokens_per_ep_rank=[32747, 32767, 32873, 32511, 32483, 32681, 32986, 33096] avg=32768.0000 max=33096 max_rank=7 min=32483 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78 max_load_reduction_pct=0.002351 moved_to_replicas=78 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=43 tokens_per_ep_rank=[33188, 32828, 32508, 32733, 32738, 32949, 32827, 32373] avg=32768.0000 max=33188 max_rank=0 min=32373 min_rank=7 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=43 tokens_per_ep_rank=[33096, 32828, 32508, 32733, 32738, 32949, 32827, 32465] avg=32768.0000 max=33096 max_rank=0 min=32465 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=44 tokens_per_ep_rank=[32713, 32821, 32610, 32707, 32812, 32536, 32952, 32993] avg=32768.0000 max=32993 max_rank=7 min=32536 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=45 tokens_per_ep_rank=[32642, 32898, 32695, 32709, 32536, 32744, 32883, 33037] avg=32768.0000 max=33037 max_rank=7 min=32536 min_rank=4 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=46 tokens_per_ep_rank=[32730, 32979, 32576, 32762, 32971, 32900, 32396, 32830] avg=32768.0000 max=32979 max_rank=1 min=32396 min_rank=6 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=47 tokens_per_ep_rank=[32774, 32951, 32513, 32794, 32998, 32617, 32717, 32780] avg=32768.0000 max=32998 max_rank=4 min=32513 min_rank=2 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=48 tokens_per_ep_rank=[32594, 32802, 32690, 32965, 32929, 32787, 32686, 32691] avg=32768.0000 max=32965 max_rank=3 min=32594 min_rank=0 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=48 total_expected_recv=1573930 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341248 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=48 total_expected_recv=1571754 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344320 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=48 total_expected_recv=1572698 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343168 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=48 total_expected_recv=1572799 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344448 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=48 total_expected_recv=1571998 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346496 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=48 total_expected_recv=1571651 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345600 peak_demand=33057 required_capacity=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=48 total_expected_recv=1573418 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344064 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=48 total_expected_recv=1574664 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=340864 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:48:22] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 707.2 | throughput per GPU (TFLOP/s/GPU): 126.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.792437E+00 | loss scale: 1.0 | grad norm: 3.107 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33002 expected_overflow=0 exact_recv_per_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33002 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32897 expected_overflow=0 exact_recv_per_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32539 expected_overflow=0 exact_recv_per_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32897 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32450 expected_overflow=0 exact_recv_per_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32539 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32450 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32489 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32489 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32433 expected_overflow=0 exact_recv_per_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32444 expected_overflow=0 exact_recv_per_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32433 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32444 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33051 expected_overflow=0 exact_recv_per_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32404 expected_overflow=0 exact_recv_per_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=33051 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32404 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32478 expected_overflow=0 exact_recv_per_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32478 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32497 expected_overflow=0 exact_recv_per_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32497 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32411 expected_overflow=0 exact_recv_per_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32411 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32181 expected_overflow=0 exact_recv_per_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33026 expected_overflow=0 exact_recv_per_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32181 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33026 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32926 expected_overflow=0 exact_recv_per_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32926 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32950 expected_overflow=0 exact_recv_per_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32950 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32975 expected_overflow=0 exact_recv_per_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32975 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32479 expected_overflow=0 exact_recv_per_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33060 expected_overflow=0 exact_recv_per_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32479 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=40960 expected_recv=33060 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32458 expected_overflow=0 exact_recv_per_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32458 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33044 expected_overflow=0 exact_recv_per_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=40960 expected_recv=33044 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32358 expected_overflow=0 exact_recv_per_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32884 expected_overflow=0 exact_recv_per_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32358 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32884 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32814 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32695 expected_overflow=0 exact_recv_per_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32646 expected_overflow=0 exact_recv_per_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32695 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=40960 expected_recv=32646 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32874 expected_overflow=0 exact_recv_per_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32874 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32952 expected_overflow=0 exact_recv_per_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=40960 expected_recv=32952 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32756 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32599 expected_overflow=0 exact_recv_per_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32599 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32636 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32992 expected_overflow=0 exact_recv_per_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32554 expected_overflow=0 exact_recv_per_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32992 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32554 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32877 expected_overflow=0 exact_recv_per_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=40960 expected_recv=32877 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32637 expected_overflow=0 exact_recv_per_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32637 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32852 expected_overflow=0 exact_recv_per_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33083 expected_overflow=0 exact_recv_per_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32852 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=40960 expected_recv=33083 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32674 expected_overflow=0 exact_recv_per_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32674 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32832 expected_overflow=0 exact_recv_per_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32832 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33029 expected_overflow=0 exact_recv_per_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=23 calls=1 budget=40960 expected_recv=33029 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32556 expected_overflow=0 exact_recv_per_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32556 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=23 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=23 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32480 expected_overflow=0 exact_recv_per_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32480 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32927 expected_overflow=0 exact_recv_per_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=24 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32927 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=24 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33002 expected_overflow=0 exact_recv_per_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32633 expected_overflow=0 exact_recv_per_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=26 calls=1 budget=40960 expected_recv=33002 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=25 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32633 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=25 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32586 expected_overflow=0 exact_recv_per_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32586 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=26 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=26 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32499 expected_overflow=0 exact_recv_per_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=27 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32499 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=27 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32604 expected_overflow=0 exact_recv_per_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32604 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32526 expected_overflow=0 exact_recv_per_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=28 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32526 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=28 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32945 expected_overflow=0 exact_recv_per_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33049 expected_overflow=0 exact_recv_per_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32945 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=30 calls=1 budget=40960 expected_recv=33049 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=29 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=29 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32498 expected_overflow=0 exact_recv_per_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32498 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=32 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32585 expected_overflow=0 exact_recv_per_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=30 calls=1 budget=40960 expected_recv=32585 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=30 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32653 expected_overflow=0 exact_recv_per_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32581 expected_overflow=0 exact_recv_per_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32653 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32581 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32910 expected_overflow=0 exact_recv_per_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=31 calls=1 budget=40960 expected_recv=32910 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=31 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=34 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32919 expected_overflow=0 exact_recv_per_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32614 expected_overflow=0 exact_recv_per_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32931 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32595 expected_overflow=0 exact_recv_per_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32919 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32614 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32595 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=32 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=32 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32964 expected_overflow=0 exact_recv_per_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=33 calls=1 budget=40960 expected_recv=32964 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33073 expected_overflow=0 exact_recv_per_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=33 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33073 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32802 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=34 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33026 expected_overflow=0 exact_recv_per_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=34 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=36 calls=1 budget=40960 expected_recv=33026 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32446 expected_overflow=0 exact_recv_per_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32446 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=37 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32475 expected_overflow=0 exact_recv_per_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=35 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32475 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=35 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32576 expected_overflow=0 exact_recv_per_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=36 calls=1 budget=40960 expected_recv=32576 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=36 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=38 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=37 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=37 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32858 expected_overflow=0 exact_recv_per_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32858 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32661 expected_overflow=0 exact_recv_per_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32661 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32527 expected_overflow=0 exact_recv_per_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=38 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32527 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=38 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32571 expected_overflow=0 exact_recv_per_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33009 expected_overflow=0 exact_recv_per_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33069 expected_overflow=0 exact_recv_per_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32563 expected_overflow=0 exact_recv_per_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32571 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=40 calls=1 budget=40960 expected_recv=33009 actual_valid_recv=34304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=41 calls=1 budget=40960 expected_recv=33069 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32563 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=34304 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32571 expected_overflow=0 exact_recv_per_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=39 calls=1 budget=40960 expected_recv=32571 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=39 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32975 expected_overflow=0 exact_recv_per_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=40 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32975 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33073 expected_overflow=0 exact_recv_per_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=40 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=42 calls=1 budget=40960 expected_recv=33073 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33037 expected_overflow=0 exact_recv_per_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=41 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=44 calls=1 budget=40960 expected_recv=33037 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=41 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32356 expected_overflow=0 exact_recv_per_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32356 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32941 expected_overflow=0 exact_recv_per_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=42 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32941 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=42 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32420 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=45 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=43 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=43 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32989 expected_overflow=0 exact_recv_per_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32989 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34176 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32960 expected_overflow=0 exact_recv_per_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=44 calls=1 budget=40960 expected_recv=32960 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=44 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=34048 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32593 expected_overflow=0 exact_recv_per_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32593 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=45 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=45 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32411 expected_overflow=0 exact_recv_per_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=46 calls=1 budget=40960 expected_recv=32411 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=46 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33152, 32450, 32780, 32424, 32897, 33011, 32539] avg=32768.0000 max=33152 max_rank=1 min=32424 min_rank=4 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33152, 32450, 32780, 32424, 32897, 33011, 32539] avg=32768.0000 max=33152 max_rank=1 min=32424 min_rank=4 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33152, 32450, 32780, 32424, 32897, 33011, 32539] avg=32768.0000 max=33152 max_rank=1 min=32424 min_rank=4 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] avg=32768.0000 max=33096 max_rank=1 min=32450 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] avg=32768.0000 max=33096 max_rank=1 min=32450 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=47 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] avg=32768.0000 max=33096 max_rank=1 min=32450 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33152, 32450, 32780, 32424, 32897, 33011, 32539] avg=32768.0000 max=33152 max_rank=1 min=32424 min_rank=4 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33152, 32450, 32780, 32424, 32897, 33011, 32539] avg=32768.0000 max=33152 max_rank=1 min=32424 min_rank=4 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=47 calls=1 received_num_tokens=40960 valid_num_tokens=33920 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32492, 32736, 32757, 32623, 32879, 33172, 32747] avg=32768.0000 max=33172 max_rank=6 min=32492 min_rank=1 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33152, 32450, 32780, 32424, 32897, 33011, 32539] avg=32768.0000 max=33152 max_rank=1 min=32424 min_rank=4 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32492, 32736, 32757, 32623, 32879, 33172, 32747] avg=32768.0000 max=33172 max_rank=6 min=32492 min_rank=1 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] avg=32768.0000 max=33096 max_rank=1 min=32450 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] avg=32768.0000 max=33096 max_rank=1 min=32450 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32492, 32736, 32757, 32623, 32879, 33172, 32747] avg=32768.0000 max=33172 max_rank=6 min=32492 min_rank=1 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33152, 32450, 32780, 32424, 32897, 33011, 32539] avg=32768.0000 max=33152 max_rank=1 min=32424 min_rank=4 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] avg=32768.0000 max=33096 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] avg=32768.0000 max=33096 max_rank=1 min=32450 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] avg=32768.0000 max=33096 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] avg=32768.0000 max=33096 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32492, 32736, 32757, 32623, 32879, 33172, 32747] avg=32768.0000 max=33172 max_rank=6 min=32492 min_rank=1 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32492, 32736, 32757, 32623, 32879, 33172, 32747] avg=32768.0000 max=33172 max_rank=6 min=32492 min_rank=1 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] avg=32768.0000 max=33096 max_rank=1 min=32450 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32803, 33002, 32687, 32645, 32672, 32764, 32819, 32752] avg=32768.0000 max=33002 max_rank=1 min=32645 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33143, 32680, 32923, 32748, 32762, 32442, 32650] avg=32768.0000 max=33143 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] avg=32768.0000 max=33096 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33143, 32680, 32923, 32748, 32762, 32442, 32650] avg=32768.0000 max=33143 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32492, 32736, 32757, 32623, 32879, 33172, 32747] avg=32768.0000 max=33172 max_rank=6 min=32492 min_rank=1 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] avg=32768.0000 max=33096 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33143, 32680, 32923, 32748, 32762, 32442, 32650] avg=32768.0000 max=33143 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] avg=32768.0000 max=33096 max_rank=1 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32492, 32736, 32757, 32623, 32879, 33172, 32747] avg=32768.0000 max=33172 max_rank=6 min=32492 min_rank=1 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] avg=32768.0000 max=33096 max_rank=1 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] avg=32768.0000 max=33096 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33152, 32450, 32780, 32424, 32897, 33011, 32539] avg=32768.0000 max=33152 max_rank=1 min=32424 min_rank=4 imbalance_ratio=1.011719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] avg=32768.0000 max=33096 max_rank=1 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33143, 32680, 32923, 32748, 32762, 32442, 32650] avg=32768.0000 max=33143 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33143, 32680, 32923, 32748, 32762, 32442, 32650] avg=32768.0000 max=33143 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] avg=32768.0000 max=33096 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32891, 33096, 32450, 32780, 32480, 32897, 33011, 32539] avg=32768.0000 max=33096 max_rank=1 min=32450 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=56 max_load_reduction_pct=0.001689 moved_to_replicas=56 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] avg=32768.0000 max=33096 max_rank=1 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33143, 32680, 32923, 32748, 32762, 32442, 32650] avg=32768.0000 max=33143 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] avg=32768.0000 max=33096 max_rank=1 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33143, 32680, 32923, 32748, 32762, 32442, 32650] avg=32768.0000 max=33143 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32492, 32736, 32757, 32623, 32879, 33172, 32747] avg=32768.0000 max=33172 max_rank=6 min=32492 min_rank=1 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] avg=32768.0000 max=33096 max_rank=1 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] avg=32768.0000 max=33096 max_rank=1 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32396, 33104, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33104 max_rank=1 min=32396 min_rank=0 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32738, 32568, 32736, 32757, 32623, 32879, 33096, 32747] avg=32768.0000 max=33096 max_rank=6 min=32568 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32396, 33104, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33104 max_rank=1 min=32396 min_rank=0 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[32396, 33104, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33104 max_rank=1 min=32396 min_rank=0 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33096 max_rank=1 min=32404 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000242 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33096 max_rank=1 min=32404 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000242 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33096 max_rank=1 min=32404 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000242 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33143, 32680, 32923, 32748, 32762, 32442, 32650] avg=32768.0000 max=33143 max_rank=1 min=32442 min_rank=6 imbalance_ratio=1.011444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32396, 33104, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33104 max_rank=1 min=32396 min_rank=0 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32396, 33104, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33104 max_rank=1 min=32396 min_rank=0 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32796, 33096, 32680, 32923, 32748, 32762, 32489, 32650] avg=32768.0000 max=33096 max_rank=1 min=32489 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=47 max_load_reduction_pct=0.001418 moved_to_replicas=47 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33096 max_rank=1 min=32404 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000242 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32396, 33104, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33104 max_rank=1 min=32396 min_rank=0 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33096 max_rank=1 min=32404 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000242 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32396, 33104, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33104 max_rank=1 min=32396 min_rank=0 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33096 max_rank=1 min=32404 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000242 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33096 max_rank=1 min=32404 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000242 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32865, 32433, 32908, 32778, 33051, 32896, 32769, 32444] avg=32768.0000 max=33051 max_rank=4 min=32433 min_rank=1 imbalance_ratio=1.008636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32396, 33104, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33104 max_rank=1 min=32396 min_rank=0 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32404, 33096, 32864, 33034, 32861, 32603, 32478, 32804] avg=32768.0000 max=33096 max_rank=1 min=32404 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000242 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32708, 32800, 32666, 32888, 32965, 32672, 32756, 32689] avg=32768.0000 max=32965 max_rank=4 min=32666 min_rank=2 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32497, 32730, 33034, 32789, 32662, 32755, 32793, 32884] avg=32768.0000 max=33034 max_rank=2 min=32497 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32814, 32411, 32733, 32861, 32944, 32825, 32658, 32898] avg=32768.0000 max=32944 max_rank=4 min=32411 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33288, 32638, 32266] avg=32768.0000 max=33288 max_rank=5 min=32266 min_rank=7 imbalance_ratio=1.015869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33288, 32638, 32266] avg=32768.0000 max=33288 max_rank=5 min=32266 min_rank=7 imbalance_ratio=1.015869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] avg=32768.0000 max=33096 max_rank=5 min=32458 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=192 max_load_reduction_pct=0.005768 moved_to_replicas=192 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33288, 32638, 32266] avg=32768.0000 max=33288 max_rank=5 min=32266 min_rank=7 imbalance_ratio=1.015869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] avg=32768.0000 max=33096 max_rank=5 min=32458 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=192 max_load_reduction_pct=0.005768 moved_to_replicas=192 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[32181, 32638, 32637, 32848, 32955, 33026, 32874, 32985] avg=32768.0000 max=33026 max_rank=5 min=32181 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] avg=32768.0000 max=33096 max_rank=5 min=32458 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=192 max_load_reduction_pct=0.005768 moved_to_replicas=192 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33288, 32638, 32266] avg=32768.0000 max=33288 max_rank=5 min=32266 min_rank=7 imbalance_ratio=1.015869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33288, 32638, 32266] avg=32768.0000 max=33288 max_rank=5 min=32266 min_rank=7 imbalance_ratio=1.015869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] avg=32768.0000 max=33096 max_rank=5 min=32458 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=192 max_load_reduction_pct=0.005768 moved_to_replicas=192 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33288, 32638, 32266] avg=32768.0000 max=33288 max_rank=5 min=32266 min_rank=7 imbalance_ratio=1.015869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32926, 32832, 32588, 32581, 32735, 32789, 32950, 32743] avg=32768.0000 max=32950 max_rank=6 min=32581 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] avg=32768.0000 max=33096 max_rank=5 min=32458 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=192 max_load_reduction_pct=0.005768 moved_to_replicas=192 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] avg=32768.0000 max=33096 max_rank=5 min=32458 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=192 max_load_reduction_pct=0.005768 moved_to_replicas=192 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33288, 32638, 32266] avg=32768.0000 max=33288 max_rank=5 min=32266 min_rank=7 imbalance_ratio=1.015869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] avg=32768.0000 max=33096 max_rank=5 min=32458 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=192 max_load_reduction_pct=0.005768 moved_to_replicas=192 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32919, 33060, 32777, 32458, 32479, 32975, 32549, 32927] avg=32768.0000 max=33060 max_rank=1 min=32458 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33288, 32638, 32266] avg=32768.0000 max=33288 max_rank=5 min=32266 min_rank=7 imbalance_ratio=1.015869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32659, 32651, 32808, 32943, 32891, 33096, 32638, 32458] avg=32768.0000 max=33096 max_rank=5 min=32458 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=192 max_load_reduction_pct=0.005768 moved_to_replicas=192 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33177, 32646, 32645, 32452, 32558, 32875] avg=32768.0000 max=33177 max_rank=2 min=32452 min_rank=5 imbalance_ratio=1.012482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33177, 32646, 32645, 32452, 32558, 32875] avg=32768.0000 max=33177 max_rank=2 min=32452 min_rank=5 imbalance_ratio=1.012482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33177, 32646, 32645, 32452, 32558, 32875] avg=32768.0000 max=33177 max_rank=2 min=32452 min_rank=5 imbalance_ratio=1.012482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=81 max_load_reduction_pct=0.002441 moved_to_replicas=81 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=81 max_load_reduction_pct=0.002441 moved_to_replicas=81 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=81 max_load_reduction_pct=0.002441 moved_to_replicas=81 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[32577, 32738, 32358, 32813, 32884, 32665, 33065, 33044] avg=32768.0000 max=33065 max_rank=6 min=32358 min_rank=2 imbalance_ratio=1.009064 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33177, 32646, 32645, 32452, 32558, 32875] avg=32768.0000 max=33177 max_rank=2 min=32452 min_rank=5 imbalance_ratio=1.012482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=81 max_load_reduction_pct=0.002441 moved_to_replicas=81 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33177, 32646, 32645, 32452, 32558, 32875] avg=32768.0000 max=33177 max_rank=2 min=32452 min_rank=5 imbalance_ratio=1.012482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33177, 32646, 32645, 32452, 32558, 32875] avg=32768.0000 max=33177 max_rank=2 min=32452 min_rank=5 imbalance_ratio=1.012482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=81 max_load_reduction_pct=0.002441 moved_to_replicas=81 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32877, 32697, 32664, 32760, 32972, 32658, 32814, 32702] avg=32768.0000 max=32972 max_rank=4 min=32658 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=81 max_load_reduction_pct=0.002441 moved_to_replicas=81 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33177, 32646, 32645, 32452, 32558, 32875] avg=32768.0000 max=33177 max_rank=2 min=32452 min_rank=5 imbalance_ratio=1.012482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=81 max_load_reduction_pct=0.002441 moved_to_replicas=81 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[32680, 32656, 32788, 32928, 32748, 32591, 32944, 32809] avg=32768.0000 max=32944 max_rank=6 min=32591 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[33115, 32636, 32866, 32877, 32599, 32596, 32756, 32699] avg=32768.0000 max=33115 max_rank=0 min=32596 min_rank=5 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[33115, 32636, 32866, 32877, 32599, 32596, 32756, 32699] avg=32768.0000 max=33115 max_rank=0 min=32596 min_rank=5 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[33115, 32636, 32866, 32877, 32599, 32596, 32756, 32699] avg=32768.0000 max=33115 max_rank=0 min=32596 min_rank=5 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] avg=32768.0000 max=33096 max_rank=0 min=32599 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] avg=32768.0000 max=33096 max_rank=0 min=32599 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] avg=32768.0000 max=33096 max_rank=0 min=32599 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33177, 32646, 32645, 32452, 32558, 32875] avg=32768.0000 max=33177 max_rank=2 min=32452 min_rank=5 imbalance_ratio=1.012482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[33115, 32636, 32866, 32877, 32599, 32596, 32756, 32699] avg=32768.0000 max=33115 max_rank=0 min=32596 min_rank=5 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[32838, 32953, 33096, 32646, 32645, 32533, 32558, 32875] avg=32768.0000 max=33096 max_rank=2 min=32533 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=81 max_load_reduction_pct=0.002441 moved_to_replicas=81 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] avg=32768.0000 max=33096 max_rank=0 min=32599 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[33115, 32636, 32866, 32877, 32599, 32596, 32756, 32699] avg=32768.0000 max=33115 max_rank=0 min=32596 min_rank=5 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[33115, 32636, 32866, 32877, 32599, 32596, 32756, 32699] avg=32768.0000 max=33115 max_rank=0 min=32596 min_rank=5 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] avg=32768.0000 max=33096 max_rank=0 min=32599 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] avg=32768.0000 max=33096 max_rank=0 min=32599 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[32695, 32685, 32825, 32952, 32963, 32669, 32738, 32617] avg=32768.0000 max=32963 max_rank=4 min=32617 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[33115, 32636, 32866, 32877, 32599, 32596, 32756, 32699] avg=32768.0000 max=33115 max_rank=0 min=32596 min_rank=5 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] avg=32768.0000 max=33096 max_rank=0 min=32599 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[32765, 32740, 32795, 32724, 32684, 32874, 32813, 32749] avg=32768.0000 max=32874 max_rank=5 min=32684 min_rank=4 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[33115, 32636, 32866, 32877, 32599, 32596, 32756, 32699] avg=32768.0000 max=33115 max_rank=0 min=32596 min_rank=5 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[33096, 32636, 32866, 32877, 32599, 32615, 32756, 32699] avg=32768.0000 max=33096 max_rank=0 min=32599 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[32992, 32751, 32819, 32745, 32786, 32614, 32883, 32554] avg=32768.0000 max=32992 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[32828, 33083, 32634, 32648, 32852, 32774, 32637, 32688] avg=32768.0000 max=33083 max_rank=1 min=32634 min_rank=2 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[32673, 32977, 32714, 32683, 32853, 32803, 32674, 32767] avg=32768.0000 max=32977 max_rank=1 min=32673 min_rank=0 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[32879, 32689, 32665, 32844, 32650, 32832, 32556, 33029] avg=32768.0000 max=33029 max_rank=7 min=32556 min_rank=6 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=25 tokens_per_ep_rank=[32921, 32734, 32505, 32865, 32651, 32824, 32786, 32858] avg=32768.0000 max=32921 max_rank=0 min=32505 min_rank=2 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=26 tokens_per_ep_rank=[32850, 32899, 32785, 32788, 32927, 32640, 32480, 32775] avg=32768.0000 max=32927 max_rank=4 min=32480 min_rank=6 imbalance_ratio=1.004852 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33172, 32450, 32868] avg=32768.0000 max=33172 max_rank=5 min=32450 min_rank=6 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33172, 32450, 32868] avg=32768.0000 max=33172 max_rank=5 min=32450 min_rank=6 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33172, 32450, 32868] avg=32768.0000 max=33172 max_rank=5 min=32450 min_rank=6 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=27 tokens_per_ep_rank=[32815, 33002, 32709, 32697, 32678, 32631, 32866, 32746] avg=32768.0000 max=33002 max_rank=1 min=32631 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] avg=32768.0000 max=33096 max_rank=5 min=32518 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] avg=32768.0000 max=33096 max_rank=5 min=32518 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] avg=32768.0000 max=33096 max_rank=5 min=32518 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33172, 32450, 32868] avg=32768.0000 max=33172 max_rank=5 min=32450 min_rank=6 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] avg=32768.0000 max=33096 max_rank=5 min=32518 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=28 tokens_per_ep_rank=[32831, 32586, 32766, 32739, 32943, 32633, 32899, 32747] avg=32768.0000 max=32943 max_rank=4 min=32586 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33172, 32450, 32868] avg=32768.0000 max=33172 max_rank=5 min=32450 min_rank=6 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33172, 32450, 32868] avg=32768.0000 max=33172 max_rank=5 min=32450 min_rank=6 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] avg=32768.0000 max=33096 max_rank=5 min=32518 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] avg=32768.0000 max=33096 max_rank=5 min=32518 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=29 tokens_per_ep_rank=[32775, 32801, 32641, 32827, 32499, 32751, 32887, 32963] avg=32768.0000 max=32963 max_rank=7 min=32499 min_rank=4 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33172, 32450, 32868] avg=32768.0000 max=33172 max_rank=5 min=32450 min_rank=6 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] avg=32768.0000 max=33096 max_rank=5 min=32518 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=30 tokens_per_ep_rank=[32785, 32714, 32604, 32678, 32677, 32820, 32972, 32894] avg=32768.0000 max=32972 max_rank=6 min=32604 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33119, 32919, 32591] avg=32768.0000 max=33119 max_rank=5 min=32591 min_rank=7 imbalance_ratio=1.010712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33119, 32919, 32591] avg=32768.0000 max=33119 max_rank=5 min=32591 min_rank=7 imbalance_ratio=1.010712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33172, 32450, 32868] avg=32768.0000 max=33172 max_rank=5 min=32450 min_rank=6 imbalance_ratio=1.012329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33119, 32919, 32591] avg=32768.0000 max=33119 max_rank=5 min=32591 min_rank=7 imbalance_ratio=1.010712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] avg=32768.0000 max=33096 max_rank=5 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23 max_load_reduction_pct=0.000694 moved_to_replicas=23 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] avg=32768.0000 max=33096 max_rank=5 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23 max_load_reduction_pct=0.000694 moved_to_replicas=23 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=31 tokens_per_ep_rank=[32752, 33049, 32750, 32585, 32518, 33096, 32526, 32868] avg=32768.0000 max=33096 max_rank=5 min=32518 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=76 max_load_reduction_pct=0.002291 moved_to_replicas=76 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] avg=32768.0000 max=33096 max_rank=5 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23 max_load_reduction_pct=0.000694 moved_to_replicas=23 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33119, 32919, 32591] avg=32768.0000 max=33119 max_rank=5 min=32591 min_rank=7 imbalance_ratio=1.010712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] avg=32768.0000 max=33096 max_rank=5 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23 max_load_reduction_pct=0.000694 moved_to_replicas=23 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=32 tokens_per_ep_rank=[32781, 32687, 32498, 32910, 32940, 32945, 32632, 32751] avg=32768.0000 max=32945 max_rank=5 min=32498 min_rank=2 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33119, 32919, 32591] avg=32768.0000 max=33119 max_rank=5 min=32591 min_rank=7 imbalance_ratio=1.010712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33119, 32919, 32591] avg=32768.0000 max=33119 max_rank=5 min=32591 min_rank=7 imbalance_ratio=1.010712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] avg=32768.0000 max=33096 max_rank=5 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23 max_load_reduction_pct=0.000694 moved_to_replicas=23 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] avg=32768.0000 max=33096 max_rank=5 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23 max_load_reduction_pct=0.000694 moved_to_replicas=23 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=33 tokens_per_ep_rank=[32913, 32735, 32653, 32671, 32630, 33096, 32667, 32779] avg=32768.0000 max=33096 max_rank=5 min=32630 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33119, 32919, 32591] avg=32768.0000 max=33119 max_rank=5 min=32591 min_rank=7 imbalance_ratio=1.010712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] avg=32768.0000 max=33096 max_rank=5 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23 max_load_reduction_pct=0.000694 moved_to_replicas=23 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=34 tokens_per_ep_rank=[32752, 32931, 32971, 32964, 32572, 32581, 32808, 32565] avg=32768.0000 max=32971 max_rank=2 min=32565 min_rank=7 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33119, 32919, 32591] avg=32768.0000 max=33119 max_rank=5 min=32591 min_rank=7 imbalance_ratio=1.010712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=35 tokens_per_ep_rank=[32595, 32859, 32808, 32594, 32659, 33096, 32919, 32614] avg=32768.0000 max=33096 max_rank=5 min=32594 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23 max_load_reduction_pct=0.000694 moved_to_replicas=23 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=36 tokens_per_ep_rank=[32977, 32802, 32627, 32712, 32638, 32719, 32887, 32782] avg=32768.0000 max=32977 max_rank=0 min=32627 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=37 tokens_per_ep_rank=[32731, 32686, 33026, 32576, 32446, 33073, 32867, 32739] avg=32768.0000 max=33073 max_rank=5 min=32446 min_rank=4 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=38 tokens_per_ep_rank=[32440, 32755, 32888, 32769, 32631, 32780, 32858, 33023] avg=32768.0000 max=33023 max_rank=7 min=32440 min_rank=0 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=39 tokens_per_ep_rank=[32711, 32921, 32815, 33035, 32867, 32475, 32638, 32682] avg=32768.0000 max=33035 max_rank=3 min=32475 min_rank=5 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=40 tokens_per_ep_rank=[32680, 32876, 32774, 32571, 32939, 32892, 32745, 32667] avg=32768.0000 max=32939 max_rank=4 min=32571 min_rank=3 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=41 tokens_per_ep_rank=[32661, 33009, 32558, 32749, 32571, 32875, 32858, 32863] avg=32768.0000 max=33009 max_rank=1 min=32558 min_rank=2 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=46 tokens_per_ep_rank=[33278, 32689, 32580, 32431, 32596, 32986, 32680, 32904] avg=32768.0000 max=33278 max_rank=0 min=32431 min_rank=3 imbalance_ratio=1.015564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=46 tokens_per_ep_rank=[33278, 32689, 32580, 32431, 32596, 32986, 32680, 32904] avg=32768.0000 max=33278 max_rank=0 min=32431 min_rank=3 imbalance_ratio=1.015564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=46 tokens_per_ep_rank=[33278, 32689, 32580, 32431, 32596, 32986, 32680, 32904] avg=32768.0000 max=33278 max_rank=0 min=32431 min_rank=3 imbalance_ratio=1.015564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=46 tokens_per_ep_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] avg=32768.0000 max=33096 max_rank=0 min=32580 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=182 max_load_reduction_pct=0.005469 moved_to_replicas=182 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=46 tokens_per_ep_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] avg=32768.0000 max=33096 max_rank=0 min=32580 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=182 max_load_reduction_pct=0.005469 moved_to_replicas=182 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=46 tokens_per_ep_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] avg=32768.0000 max=33096 max_rank=0 min=32580 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=182 max_load_reduction_pct=0.005469 moved_to_replicas=182 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=42 tokens_per_ep_rank=[32693, 32808, 32759, 32996, 32729, 32527, 33069, 32563] avg=32768.0000 max=33069 max_rank=6 min=32527 min_rank=5 imbalance_ratio=1.009186 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=46 tokens_per_ep_rank=[33278, 32689, 32580, 32431, 32596, 32986, 32680, 32904] avg=32768.0000 max=33278 max_rank=0 min=32431 min_rank=3 imbalance_ratio=1.015564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=46 tokens_per_ep_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] avg=32768.0000 max=33096 max_rank=0 min=32580 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=182 max_load_reduction_pct=0.005469 moved_to_replicas=182 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=43 tokens_per_ep_rank=[32766, 32642, 32975, 32649, 33073, 32708, 32568, 32763] avg=32768.0000 max=33073 max_rank=4 min=32568 min_rank=6 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=46 tokens_per_ep_rank=[33278, 32689, 32580, 32431, 32596, 32986, 32680, 32904] avg=32768.0000 max=33278 max_rank=0 min=32431 min_rank=3 imbalance_ratio=1.015564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=46 tokens_per_ep_rank=[33278, 32689, 32580, 32431, 32596, 32986, 32680, 32904] avg=32768.0000 max=33278 max_rank=0 min=32431 min_rank=3 imbalance_ratio=1.015564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=46 tokens_per_ep_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] avg=32768.0000 max=33096 max_rank=0 min=32580 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=182 max_load_reduction_pct=0.005469 moved_to_replicas=182 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=46 tokens_per_ep_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] avg=32768.0000 max=33096 max_rank=0 min=32580 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=182 max_load_reduction_pct=0.005469 moved_to_replicas=182 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=46 tokens_per_ep_rank=[33278, 32689, 32580, 32431, 32596, 32986, 32680, 32904] avg=32768.0000 max=33278 max_rank=0 min=32431 min_rank=3 imbalance_ratio=1.015564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=44 tokens_per_ep_rank=[32840, 32716, 32795, 32752, 32656, 32929, 32847, 32609] avg=32768.0000 max=32929 max_rank=5 min=32609 min_rank=7 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=46 tokens_per_ep_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] avg=32768.0000 max=33096 max_rank=0 min=32580 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=182 max_load_reduction_pct=0.005469 moved_to_replicas=182 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=45 tokens_per_ep_rank=[32731, 32420, 32941, 32960, 32807, 33037, 32356, 32892] avg=32768.0000 max=33037 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.008209 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=46 tokens_per_ep_rank=[33278, 32689, 32580, 32431, 32596, 32986, 32680, 32904] avg=32768.0000 max=33278 max_rank=0 min=32431 min_rank=3 imbalance_ratio=1.015564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=46 tokens_per_ep_rank=[33096, 32689, 32580, 32613, 32596, 32986, 32680, 32904] avg=32768.0000 max=33096 max_rank=0 min=32580 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=182 max_load_reduction_pct=0.005469 moved_to_replicas=182 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=47 tokens_per_ep_rank=[32790, 32706, 32848, 32411, 32593, 32946, 32861, 32989] avg=32768.0000 max=32989 max_rank=7 min=32411 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=48 tokens_per_ep_rank=[32782, 32823, 32765, 32842, 32681, 32922, 32721, 32608] avg=32768.0000 max=32922 max_rank=5 min=32608 min_rank=7 imbalance_ratio=1.004700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=48 total_expected_recv=1574549 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341504 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=48 total_expected_recv=1572697 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343424 peak_demand=33044 required_capacity=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=48 total_expected_recv=1572715 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343680 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=48 total_expected_recv=1571833 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=345856 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=48 total_expected_recv=1571623 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346624 peak_demand=33073 required_capacity=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=48 total_expected_recv=1573868 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342784 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=48 total_expected_recv=1572769 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344192 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=48 total_expected_recv=1572858 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344320 peak_demand=33035 required_capacity=1.008148 + [2026-08-26 15:48:23] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 706.5 | throughput per GPU (TFLOP/s/GPU): 126.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.762493E+00 | loss scale: 1.0 | grad norm: 3.001 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34304 | min safe capacity: 1.0469 | +[after training is done] datetime: 2026-08-26 15:48:23 +[rank0]:[W826 15:48:24.455444572 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 15:48:25.929875539 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 15:48:25.932763606 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 15:48:25.933249210 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 15:48:25.933537174 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 15:48:25.934993885 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 15:48:25.935192268 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 15:48:26.130457110 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:48:27.084000 137212 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:48:27.085000 137212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 137305 closing signal SIGTERM +W0826 15:48:27.085000 137212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 137306 closing signal SIGTERM +W0826 15:48:27.086000 137212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 137307 closing signal SIGTERM +W0826 15:48:27.087000 137212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 137308 closing signal SIGTERM +W0826 15:48:27.088000 137212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 137309 closing signal SIGTERM +W0826 15:48:27.088000 137212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 137310 closing signal SIGTERM +W0826 15:48:27.089000 137212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 137311 closing signal SIGTERM +W0826 15:48:27.090000 137212 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 137312 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 137212 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index a9a0a1fb71acd0728768139446cac6c7565fd156..54cd1c0796ff4c43de5a9e36de6ef12123878348 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -1,8 +1,8 @@ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir -QWEN3_235B ultraep measure 1.25 OK 122751 1 0.0 1.01004 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 -QWEN3_235B ultraep validate 1.02 ASSERT 54649 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1 -QWEN3_30B ultraep measure 1.25 OK 117915 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 -QWEN3_30B ultraep validate 1.02 ASSERT 48657 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1 +QWEN3_235B ultraep measure 1.25 OK 122751 1 0.0 1.010056 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +QWEN3_235B ultraep forced 1.25 OK 122739 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 +QWEN3_30B ultraep measure 1.25 OK 117927 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN3_30B ultraep forced 1.25 OK 117867 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 DSV2_LITE_L24 ultraep measure 1.25 OK 130023 1 0.0 1.010071 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 -DSV2_LITE_L24 ultraep validate 1.02 ASSERT 39855 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1 -QWEN2_57B ultraep measure 1.25 FAIL 47627 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +DSV2_LITE_L24 ultraep forced 1.25 OK 129971 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 +QWEN2_57B ultraep measure 1.25 OK 97675 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index c203cb0e891ef086e0affffdee5e029192210ff9..39eea8c931163982b6059f4e4997398a4c5112da 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -1,5 +1,4 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir -QWEN3_235B ultraep ERROR_ASSERT -QWEN3_30B ultraep ERROR_ASSERT -DSV2_LITE_L24 ultraep ERROR_ASSERT -QWEN2_57B ultraep ERROR_FAIL +QWEN3_235B ultraep SELECTED 1.25 1.010056 0.0 122751 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +QWEN3_30B ultraep SELECTED 1.25 1.010101 0.0 117927 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +DSV2_LITE_L24 ultraep SELECTED 1.25 1.010071 0.0 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv index b43c1f2b0e9b0ac4fcd2b83c5f1b476438fef49b..add80251e873ecdef765198db59bd1235186bea4 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -1,8 +1,8 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir -QWEN3_235B ultraep matched 16 8192 8 1.25 none 1 OK 339.56 338.10 122751 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 -QWEN3_235B ultraep matched 16 8192 8 1.02 none 1 ASSERT NA NA 54649 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p02/rep1 -QWEN3_30B ultraep matched 16 4096 8 1.25 none 1 OK 780.66 722.00 117915 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 -QWEN3_30B ultraep matched 16 4096 8 1.02 none 1 ASSERT NA NA 48657 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p02/rep1 -DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none 1 OK 641.99 639.30 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 -DSV2_LITE_L24 ultraep matched 8 8192 8 1.02 none 1 ASSERT NA NA 39855 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p02/rep1 -QWEN2_57B ultraep matched 8 4096 8 1.25 none 1 FAIL NA NA 47627 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN3_235B ultraep matched 16 8192 8 1.25 none 1 OK 340.64 339.20 122751 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +QWEN3_235B ultraep matched 16 8192 8 1.25 none+forcebal 1 OK 317.89 317.40 122739 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 +QWEN3_30B ultraep matched 16 4096 8 1.25 none 1 OK 749.31 712.40 117927 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN3_30B ultraep matched 16 4096 8 1.25 none+forcebal 1 OK 737.56 705.60 117867 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 +DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none 1 OK 639.96 638.80 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none+forcebal 1 OK 625.83 624.60 129971 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 +QWEN2_57B ultraep matched 8 4096 8 1.25 none 1 OK 238.29 230.10 97675 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1